User Inputs

output.var = params$output.var 

transform.abs = FALSE
log.pred = params$log.pred
norm.pred = params$norm.pred
eda = params$eda
algo.forward.caret = params$algo.forward.caret
algo.backward.caret = params$algo.backward.caret
algo.stepwise.caret = params$algo.stepwise.caret
algo.LASSO.caret = params$algo.LASSO.caret
algo.LARS.caret = params$algo.LARS.caret
message("Parameters used for training/prediction: ")
## Parameters used for training/prediction:
str(params)
## List of 9
##  $ output.var         : chr "y3"
##  $ log.pred           : logi TRUE
##  $ norm.pred          : logi FALSE
##  $ eda                : logi FALSE
##  $ algo.forward.caret : logi TRUE
##  $ algo.backward.caret: logi TRUE
##  $ algo.stepwise.caret: logi TRUE
##  $ algo.LASSO.caret   : logi TRUE
##  $ algo.LARS.caret    : logi TRUE
# Setup Labels
output.var.tr = if (log.pred == TRUE)  paste0(output.var,'.log') else  output.var.tr = output.var
# output.var.tr = if (log.pred == TRUE)  paste0(output.var,'.cuberoot') else  output.var.tr = output.var
# output.var.tr = if (norm.pred == TRUE)  paste0(output.var,'.bestnorm') else  output.var.tr = output.var

Loading Data

feat  = read.csv('../../Data/features_highprec.csv')
labels = read.csv('../../Data/labels.csv')
predictors = names(dplyr::select(feat,-JobName))
data.ori = inner_join(feat,labels,by='JobName')
#data.ori = inner_join(feat,select_at(labels,c('JobName',output.var)),by='JobName')

Data validation

cc  = complete.cases(data.ori)
data.notComplete = data.ori[! cc,]
data = data.ori[cc,] %>% select_at(c(predictors,output.var,'JobName'))
message('Original cases: ',nrow(data.ori))
## Original cases: 10000
message('Non-Complete cases: ',nrow(data.notComplete))
## Non-Complete cases: 3020
message('Complete cases: ',nrow(data))
## Complete cases: 6980
summary(dplyr::select_at(data,c('JobName',output.var)))
##       JobName           y3        
##  Job_00001:   1   Min.   : 95.91  
##  Job_00002:   1   1st Qu.:118.29  
##  Job_00003:   1   Median :124.03  
##  Job_00004:   1   Mean   :125.40  
##  Job_00007:   1   3rd Qu.:131.06  
##  Job_00008:   1   Max.   :193.73  
##  (Other)  :6974

Output Variable

The Output Variable y3 shows right skewness, so will proceed with a log transformation

Histogram

df=gather(select_at(data,output.var))
ggplot(df, aes(x=value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() 

  #stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  

QQPlot

ggplot(gather(select_at(data,output.var)), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Trasformation of Output Variable from y3 to y3.log

if(log.pred==TRUE) data[[output.var.tr]] = log(data[[output.var]],10) else
# if(log.pred==TRUE) data[[output.var.tr]] = (data[[output.var]])^(1/3) else
  data[[output.var.tr]] = data[[output.var]]
df=gather(select_at(data,c(output.var,output.var.tr)))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=2)

ggplot(gather(select_at(data,c(output.var,output.var.tr))), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Best Normalizator y3

Normalization of y3 using bestNormalize package. (suggested orderNorm) This is cool, but I think is too far for the objective of the project

if (norm.pred == TRUE){
  t=bestNormalize::bestNormalize(data[[output.var]])
  t
  qqnorm(data[[output.var]])
  qqnorm(predict(t))
  data[[output.var.tr]] = predict(t)
}

orderNorm() is a rank-based procedure by which the values of a vector are mapped to their percentile, which is then mapped to the same percentile of the normal distribution. Without the presence of ties, this essentially guarantees that the transformation leads to a uniform distribution

Predictors

Feature Engineering

data$x2byx1 = data$x2/data$x1
data$x6byx5 = data$x6/data$x5
data$x9byx7 = data$x9/data$x7
data$x10byx8 = data$x10/data$x8
data$x14byx12 = data$x14/data$x12
data$x15byx13 = data$x15/data$x13
data$x17byx16 = data$x17/data$x16
data$x19byx18 = data$x19/data$x18
data$x21byx20 = data$x21/data$x20
data$x23byx22 = data$x23/data$x22
data$x1log = log(data$x1)
data$x2log = log(data$x2)
data$x5log = log(data$x5)
data$x6log = log(data$x6)
data$x7log = log(data$x7)
data$x9log = log(data$x9)
data$x8log = log(data$x8)
data$x10log = log(data$x10)
data$x12log = log(data$x12)
data$x14log = log(data$x14)
data$x13log = log(data$x13)
data$x15log = log(data$x15)
data$x16log = log(data$x16)
data$x17log = log(data$x17)
data$x18log = log(data$x18)
data$x19log = log(data$x19)
data$x20log = log(data$x20)
data$x21log = log(data$x21)
data$x22log = log(data$x22)
data$x23log = log(data$x23)
data$x11log = log(data$x11)
data$x1sqinv = 1/(data$x1)^2 
data$x5sqinv = 1/(data$x5)^2 
data$x7sqinv = 1/(data$x7)^2 
data$x8sqinv = 1/(data$x8)^2 
data$x12sqinv = 1/(data$x12)^2 
data$x13sqinv = 1/(data$x13)^2 
data$x16sqinv = 1/(data$x16)^2 
data$x18sqinv = 1/(data$x18)^2 
data$x20sqinv = 1/(data$x20)^2 
data$x22sqinv = 1/(data$x22)^2 
print("Predictors before feature engineering")
## [1] "Predictors before feature engineering"
print (predictors)
##   [1] "x1"      "x2"      "x3"      "x4"      "x5"      "x6"      "x7"      "x8"      "x9"      "x10"     "x11"    
##  [12] "x12"     "x13"     "x14"     "x15"     "x16"     "x17"     "x18"     "x19"     "x20"     "x21"     "x22"    
##  [23] "x23"     "stat1"   "stat2"   "stat3"   "stat4"   "stat5"   "stat6"   "stat7"   "stat8"   "stat9"   "stat10" 
##  [34] "stat11"  "stat12"  "stat13"  "stat14"  "stat15"  "stat16"  "stat17"  "stat18"  "stat19"  "stat20"  "stat21" 
##  [45] "stat22"  "stat23"  "stat24"  "stat25"  "stat26"  "stat27"  "stat28"  "stat29"  "stat30"  "stat31"  "stat32" 
##  [56] "stat33"  "stat34"  "stat35"  "stat36"  "stat37"  "stat38"  "stat39"  "stat40"  "stat41"  "stat42"  "stat43" 
##  [67] "stat44"  "stat45"  "stat46"  "stat47"  "stat48"  "stat49"  "stat50"  "stat51"  "stat52"  "stat53"  "stat54" 
##  [78] "stat55"  "stat56"  "stat57"  "stat58"  "stat59"  "stat60"  "stat61"  "stat62"  "stat63"  "stat64"  "stat65" 
##  [89] "stat66"  "stat67"  "stat68"  "stat69"  "stat70"  "stat71"  "stat72"  "stat73"  "stat74"  "stat75"  "stat76" 
## [100] "stat77"  "stat78"  "stat79"  "stat80"  "stat81"  "stat82"  "stat83"  "stat84"  "stat85"  "stat86"  "stat87" 
## [111] "stat88"  "stat89"  "stat90"  "stat91"  "stat92"  "stat93"  "stat94"  "stat95"  "stat96"  "stat97"  "stat98" 
## [122] "stat99"  "stat100" "stat101" "stat102" "stat103" "stat104" "stat105" "stat106" "stat107" "stat108" "stat109"
## [133] "stat110" "stat111" "stat112" "stat113" "stat114" "stat115" "stat116" "stat117" "stat118" "stat119" "stat120"
## [144] "stat121" "stat122" "stat123" "stat124" "stat125" "stat126" "stat127" "stat128" "stat129" "stat130" "stat131"
## [155] "stat132" "stat133" "stat134" "stat135" "stat136" "stat137" "stat138" "stat139" "stat140" "stat141" "stat142"
## [166] "stat143" "stat144" "stat145" "stat146" "stat147" "stat148" "stat149" "stat150" "stat151" "stat152" "stat153"
## [177] "stat154" "stat155" "stat156" "stat157" "stat158" "stat159" "stat160" "stat161" "stat162" "stat163" "stat164"
## [188] "stat165" "stat166" "stat167" "stat168" "stat169" "stat170" "stat171" "stat172" "stat173" "stat174" "stat175"
## [199] "stat176" "stat177" "stat178" "stat179" "stat180" "stat181" "stat182" "stat183" "stat184" "stat185" "stat186"
## [210] "stat187" "stat188" "stat189" "stat190" "stat191" "stat192" "stat193" "stat194" "stat195" "stat196" "stat197"
## [221] "stat198" "stat199" "stat200" "stat201" "stat202" "stat203" "stat204" "stat205" "stat206" "stat207" "stat208"
## [232] "stat209" "stat210" "stat211" "stat212" "stat213" "stat214" "stat215" "stat216" "stat217"
controlled.vars = colnames(data)[grep("^x",colnames(data))]
stat.vars = colnames(data)[grep("^stat",colnames(data))]

predictors = c(controlled.vars,stat.vars)

print("Predictors after feature engineering")
## [1] "Predictors after feature engineering"
print (predictors)
##   [1] "x1"       "x2"       "x3"       "x4"       "x5"       "x6"       "x7"       "x8"       "x9"       "x10"     
##  [11] "x11"      "x12"      "x13"      "x14"      "x15"      "x16"      "x17"      "x18"      "x19"      "x20"     
##  [21] "x21"      "x22"      "x23"      "x2byx1"   "x6byx5"   "x9byx7"   "x10byx8"  "x14byx12" "x15byx13" "x17byx16"
##  [31] "x19byx18" "x21byx20" "x23byx22" "x1log"    "x2log"    "x5log"    "x6log"    "x7log"    "x9log"    "x8log"   
##  [41] "x10log"   "x12log"   "x14log"   "x13log"   "x15log"   "x16log"   "x17log"   "x18log"   "x19log"   "x20log"  
##  [51] "x21log"   "x22log"   "x23log"   "x11log"   "x1sqinv"  "x5sqinv"  "x7sqinv"  "x8sqinv"  "x12sqinv" "x13sqinv"
##  [61] "x16sqinv" "x18sqinv" "x20sqinv" "x22sqinv" "stat1"    "stat2"    "stat3"    "stat4"    "stat5"    "stat6"   
##  [71] "stat7"    "stat8"    "stat9"    "stat10"   "stat11"   "stat12"   "stat13"   "stat14"   "stat15"   "stat16"  
##  [81] "stat17"   "stat18"   "stat19"   "stat20"   "stat21"   "stat22"   "stat23"   "stat24"   "stat25"   "stat26"  
##  [91] "stat27"   "stat28"   "stat29"   "stat30"   "stat31"   "stat32"   "stat33"   "stat34"   "stat35"   "stat36"  
## [101] "stat37"   "stat38"   "stat39"   "stat40"   "stat41"   "stat42"   "stat43"   "stat44"   "stat45"   "stat46"  
## [111] "stat47"   "stat48"   "stat49"   "stat50"   "stat51"   "stat52"   "stat53"   "stat54"   "stat55"   "stat56"  
## [121] "stat57"   "stat58"   "stat59"   "stat60"   "stat61"   "stat62"   "stat63"   "stat64"   "stat65"   "stat66"  
## [131] "stat67"   "stat68"   "stat69"   "stat70"   "stat71"   "stat72"   "stat73"   "stat74"   "stat75"   "stat76"  
## [141] "stat77"   "stat78"   "stat79"   "stat80"   "stat81"   "stat82"   "stat83"   "stat84"   "stat85"   "stat86"  
## [151] "stat87"   "stat88"   "stat89"   "stat90"   "stat91"   "stat92"   "stat93"   "stat94"   "stat95"   "stat96"  
## [161] "stat97"   "stat98"   "stat99"   "stat100"  "stat101"  "stat102"  "stat103"  "stat104"  "stat105"  "stat106" 
## [171] "stat107"  "stat108"  "stat109"  "stat110"  "stat111"  "stat112"  "stat113"  "stat114"  "stat115"  "stat116" 
## [181] "stat117"  "stat118"  "stat119"  "stat120"  "stat121"  "stat122"  "stat123"  "stat124"  "stat125"  "stat126" 
## [191] "stat127"  "stat128"  "stat129"  "stat130"  "stat131"  "stat132"  "stat133"  "stat134"  "stat135"  "stat136" 
## [201] "stat137"  "stat138"  "stat139"  "stat140"  "stat141"  "stat142"  "stat143"  "stat144"  "stat145"  "stat146" 
## [211] "stat147"  "stat148"  "stat149"  "stat150"  "stat151"  "stat152"  "stat153"  "stat154"  "stat155"  "stat156" 
## [221] "stat157"  "stat158"  "stat159"  "stat160"  "stat161"  "stat162"  "stat163"  "stat164"  "stat165"  "stat166" 
## [231] "stat167"  "stat168"  "stat169"  "stat170"  "stat171"  "stat172"  "stat173"  "stat174"  "stat175"  "stat176" 
## [241] "stat177"  "stat178"  "stat179"  "stat180"  "stat181"  "stat182"  "stat183"  "stat184"  "stat185"  "stat186" 
## [251] "stat187"  "stat188"  "stat189"  "stat190"  "stat191"  "stat192"  "stat193"  "stat194"  "stat195"  "stat196" 
## [261] "stat197"  "stat198"  "stat199"  "stat200"  "stat201"  "stat202"  "stat203"  "stat204"  "stat205"  "stat206" 
## [271] "stat207"  "stat208"  "stat209"  "stat210"  "stat211"  "stat212"  "stat213"  "stat214"  "stat215"  "stat216" 
## [281] "stat217"

All predictors show a Fat-Tail situation, where the two tails are very tall, and a low distribution around the mean. The orderNorm transformation can help (see [Best Normalizator] section)

Interesting Predictors

Histograms

if (eda == TRUE){
  cols = c('x11','x18','stat98','x7','stat110')
  df=gather(select_at(data,cols))
  ggplot(df, aes(value)) + 
    geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
    geom_density() + 
    # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
    facet_wrap(~key, scales = 'free',ncol=3)
 
  lapply(select_at(data,cols),summary)
}

Scatter plot vs. output variable **y3.log

if (eda == TRUE){
  d = gather(dplyr::select_at(data,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
  ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
    geom_point(color='light green',alpha=0.5) + 
    geom_smooth() + 
    facet_wrap(~target, scales = 'free',ncol=3)
}

All Predictors

Histograms

All indicators have a strong indication of Fat-Tails

if (eda == TRUE){
  df=gather(select_at(data,predictors))
  ggplot(df, aes(value)) + 
    geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
    geom_density() + 
    # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
    facet_wrap(~key, scales = 'free',ncol=4)
}

Correlations

With Output Variable

if (eda == TRUE){
  #chart.Correlation(select(data,-JobName),  pch=21)
  # https://stackoverflow.com/questions/27034655/how-to-use-dplyrarrangedesc-when-using-a-string-as-column-name
  t=as.data.frame(round(cor(dplyr::select(data,-one_of(output.var.tr,'JobName'))
                            ,select_at(data,output.var.tr)),4))  %>%
    #rownames_to_column(var='variable') %>% filter(variable != !!output.var) %>% arrange(-y3.log)
    rownames_to_column(var='variable') %>% filter(variable != !!output.var) %>% arrange(-!!sym(output.var.tr))
}
if (eda == TRUE){
  message("Top Positive")
  #kable(head(arrange(t,desc(y3.log)),20))
  kable(head(arrange(t,desc(!!sym(output.var.tr))),20))
}
if (eda == TRUE){
  message("Top Negative")
  #kable(head(arrange(t,y3.log),20))
  kable(head(arrange(t,!!sym(output.var.tr)),20))  
}

Between All Variables

if (eda == TRUE){
  #chart.Correlation(select(data,-JobName),  pch=21)
  t=as.data.frame(round(cor(dplyr::select(data,-one_of('JobName'))),4))
  #DT::datatable(t,options=list(scrollX=T))
  message("Showing only 10 variables")
  kable(t[1:10,1:10])
}

Scatter Plots with Output Variable

Scatter plots with all predictors and the output variable (y3.log)

if (eda == TRUE){
  d = gather(dplyr::select_at(data,c(predictors,output.var.tr)),key=target,value=value,-!!output.var.tr)
  ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
    geom_point(color='light blue',alpha=0.5) + 
    geom_smooth() + 
    facet_wrap(~target, scales = 'free',ncol=4)
}

Multicollinearity - VIF

No Multicollinearity among predictors

Showing Top predictor by VIF Value

if (eda == TRUE){
  vifDF = usdm::vif(select_at(data,predictors)) %>% arrange(desc(VIF))
  head(vifDF,75)
}

Feature Eng

  • Square Root transformation for x18
data.tr=data %>%
  mutate(x18.sqrt = sqrt(x18)) 
cols=c('x18','x18.sqrt')

Comparing Pre and Post Transformation Density Plots

d = gather(dplyr::select_at(data.tr,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

#removing unwanted variables
data.tr=data.tr %>%
  #dplyr::select_at(names(data.tr)[! names(data.tr) %in% c('x18','y3','JobName')])
  dplyr::select_at(names(data.tr)[! names(data.tr) %in% c('JobName')])

data=data.tr
label.names=output.var.tr

Modeling

PCA

# 0 for no interaction, 
# 1 for Full 2 way interaction and 
# 2 for Selective 2 way interaction
# 3 for Selective 3 way interaction
InteractionMode = 2

pca.vars  = names(data)
pca.vars = pca.vars[!pca.vars %in% label.names]


# http://sshaikh.org/2015/05/06/parallelize-machine-learning-in-r-with-multi-core-cpus/
# #cl <- makeCluster(ceiling(detectCores()*0.5)) # use 75% of cores only, leave rest for other tasks
cl <- makeCluster(detectCores()*0.75) # use 75% of cores only, leave rest for other tasks
registerDoParallel(cl)

if(InteractionMode == 1){
  pca.formula =as.formula(paste0('~(',paste0(pca.vars, collapse ='+'),')^2'))
  pca.model = prcomp(formula=pca.formula,data=data[,pca.vars],center=T,scale.=T,retx = T)
  #saveRDS(pca.model,'pca.model.rds')
}
if (InteractionMode == 0){
  pca.model =  prcomp(x=data[,pca.vars],center=T,scale.=T,retx = T)
}
if (InteractionMode >= 2 & InteractionMode <= 3){
  controlled.vars = pca.vars[grep("^x",pca.vars)]
  stat.vars = pca.vars[grep("^stat",pca.vars)]
  
  if (InteractionMode >= 2){
    interaction.form = paste0('~(',paste0(controlled.vars, collapse ='+'),')^2')
  }
  if (InteractionMode >= 3){
    interaction.form = paste0('~(',paste0(controlled.vars, collapse ='+'),')^3')
  }
  no.interact.form = paste0(stat.vars, collapse ='+')
  
  pca.formula = as.formula(paste(interaction.form, no.interact.form, sep = "+"))
  pca.model = prcomp(formula=pca.formula,data=data[,pca.vars],center=T,scale.=T,retx = T)
}

stopCluster(cl)
registerDoSEQ() # register sequential engine in case you are not using this function anymore
targetCumVar = .9

pca.model$var = pca.model$sdev ^ 2 #eigenvalues
pca.model$pvar = pca.model$var / sum(pca.model$var)
pca.model$cumpvar = cumsum(pca.model$pvar )
pca.model$pcaSel = pca.model$cumpvar<=targetCumVar
pca.model$pcaSelCount = sum(pca.model$pcaSel)
pca.model$pcaSelTotVar = sum(pca.model$pvar[pca.model$pcaSel])
message(pca.model$pcaSelCount, " PCAs justify ",percent(targetCumVar)," of the total Variance. (",percent(pca.model$pcaSelTotVar),")")
## 164 PCAs justify 90.0% of the total Variance. (90.0%)
plot(pca.model$var,xlab="Principal component", ylab="Proportion of variance explained",   type='b')

plot(cumsum(pca.model$pvar ),xlab="Principal component", ylab="Cumulative Proportion of variance explained", ylim=c(0,1), type='b')

screeplot(pca.model,npcs = pca.model$pcaSelCount)

screeplot(pca.model,npcs = pca.model$pcaSelCount,type='lines')

#summary(pca.model)
#pca.model$rotation
#creating dataset
data.pca = dplyr::select(data,!!label.names) %>% 
  dplyr::bind_cols(dplyr::select(as.data.frame(pca.model$x)
                                 ,!!colnames(pca.model$rotation)[pca.model$pcaSel])
  )

Train Test Split

data.pca = data.pca[sample(nrow(data.pca)),] # randomly shuffle data
split = sample.split(data.pca[,label.names], SplitRatio = 0.8)

data.train = subset(data.pca, split == TRUE)
data.test = subset(data.pca, split == FALSE)

Common Functions

plot.diagnostics <-  function(model, train) {
  plot(model)
  
  residuals = resid(model) # Plotted above in plot(lm.out)
  r.standard = rstandard(model)
  r.student = rstudent(model)
  
  df = data.frame(x=predict(model,train),y=r.student)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = 0,size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  
  df = data.frame(x=predict(model,train),y=r.standard)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = c(-2,0,2),size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  # Histogram
  df=data.frame(r.student)
  p=ggplot(data=df,aes(r.student)) +
    geom_histogram(aes(y=..density..),bins = 50,fill='blue',alpha=0.6) + 
    stat_function(fun = dnorm, n = 100, args = list(mean = 0, sd = 1)) +
    ylab("Density")+
    xlab("Studentized Residuals")+
    ggtitle("Distribution of Studentized Residuals")
  plot(p)
  # http://www.stat.columbia.edu/~martin/W2024/R7.pdf
  # Influential plots
  inf.meas = influence.measures(model)
  # print (summary(inf.meas)) # too much data
  
  # Leverage plot
  lev = hat(model.matrix(model))
  df=tibble::rownames_to_column(as.data.frame(lev),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=lev)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    ylab('Leverage - check') + 
    xlab('Index')
  plot(p)
  # Cook's Distance
  cd = cooks.distance(model)
  df=tibble::rownames_to_column(as.data.frame(cd),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=cd)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_text(data=filter(df,cd>15/nrow(train)),aes(label=id),check_overlap=T,size=3,vjust=-.5)+
    ylab('Cooks distances') + 
    geom_hline(yintercept = c(4/nrow(train),0),size=1)+
    xlab('Index')
  plot(p)
  print (paste("Number of data points that have Cook's D > 4/n: ", length(cd[cd > 4/nrow(train)]), sep = "")) 
  print (paste("Number of data points that have Cook's D > 1: ", length(cd[cd > 1]), sep = "")) 
  return(cd)
}

# function to set up random seeds
# Based on http://jaehyeon-kim.github.io/2015/05/Setup-Random-Seeds-on-Caret-Package.html 
setCaretSeeds <- function(method = "cv", numbers = 1, repeats = 1, tunes = NULL, seed = 1701) {
  #B is the number of resamples and integer vector of M (numbers + tune length if any)
  B <- if (method == "cv") numbers
  else if(method == "repeatedcv") numbers * repeats
  else NULL
  if(is.null(length)) {
    seeds <- NULL
  } else {
    set.seed(seed = seed)
    seeds <- vector(mode = "list", length = B)
    seeds <- lapply(seeds, function(x) sample.int(n = 1000000
                                                  , size = numbers + ifelse(is.null(tunes), 0, tunes)))
    seeds[[length(seeds) + 1]] <- sample.int(n = 1000000, size = 1)
  }
  # return seeds
  seeds
}



train.caret.glmselect = function(formula, data, method
                                 ,subopt = NULL, feature.names
                                 , train.control = NULL, tune.grid = NULL, pre.proc = NULL){
  
  if(is.null(train.control)){
    train.control <- trainControl(method = "cv"
                              ,number = 10
                              ,seeds = setCaretSeeds(method = "cv"
                                                     , numbers = 10
                                                     , seed = 1701)
                              ,search = "grid"
                              ,verboseIter = TRUE
                              ,allowParallel = TRUE
                              )
  }
  
  if(is.null(tune.grid)){
    if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
      tune.grid = data.frame(nvmax = 1:length(feature.names))
    }
    if (method == 'glmnet' && subopt == 'LASSO'){
      # Will only show 1 Lambda value during training, but that is OK
      # https://stackoverflow.com/questions/47526544/why-need-to-tune-lambda-with-carettrain-method-glmnet-and-cv-glmnet
      # Another option for LASSO is this: https://github.com/topepo/caret/blob/master/RegressionTests/Code/lasso.R
      lambda = 10^seq(-2,0, length =100)
      alpha = c(1)
      tune.grid = expand.grid(alpha = alpha,lambda = lambda)
    }
    if (method == 'lars'){
      # https://github.com/topepo/caret/blob/master/RegressionTests/Code/lars.R
      fraction = seq(0, 1, length = 100)
      tune.grid = expand.grid(fraction = fraction)
      pre.proc = c("center", "scale") 
    }
  }
  
  # http://sshaikh.org/2015/05/06/parallelize-machine-learning-in-r-with-multi-core-cpus/
  # #cl <- makeCluster(ceiling(detectCores()*0.5)) # use 75% of cores only, leave rest for other tasks
  cl <- makeCluster(detectCores()*0.75) # use 75% of cores only, leave rest for other tasks
  registerDoParallel(cl)

  set.seed(1) 
  # note that the seed has to actually be set just before this function is called
  # settign is above just not ensure reproducibility for some reason
  model.caret <- caret::train(formula
                              , data = data
                              , method = method
                              , tuneGrid = tune.grid
                              , trControl = train.control
                              , preProc = pre.proc
                              )
  
  stopCluster(cl)
  registerDoSEQ() # register sequential engine in case you are not using this function anymore
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    print("All models results")
    print(model.caret$results) # all model results
    print("Best Model")
    print(model.caret$bestTune) # best model
    model = model.caret$finalModel

    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-nvmax) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=nvmax,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    # leap function does not support studentized residuals
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)
   
    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    id = rownames(model.caret$bestTune)    
    # Provides the coefficients of the best model
    # regsubsets doens return a full model (see documentation of regsubset), so we need to recalcualte themodel
    # https://stackoverflow.com/questions/13063762/how-to-obtain-a-lm-object-from-regsubsets
    print("Coefficients of final model:")
    coefs <- coef(model, id=id)
    #calculate the model to the the coef intervals
    nams <- names(coefs)
    nams <- nams[!nams %in% "(Intercept)"]
    response <-  as.character(formula[[2]])
    form <- as.formula(paste(response, paste(nams, collapse = " + "), sep = " ~ "))
    mod <- lm(form, data = data)
    #coefs
    #coef(mod)
    print(car::Confint(mod))
    return(list(model = model,id = id, residPlot = residPlot, residHistogram=residHistogram
                ,modelLM=mod))
  }
  if (method == 'glmnet' && subopt == 'LASSO'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    print(model.caret$results)
    model=model.caret$finalModel
    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-lambda) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=lambda,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot 
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') +
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    #no interval for glmnet: https://stackoverflow.com/questions/39750965/confidence-intervals-for-ridge-regression
    t=coef(model,s=model.caret$bestTune$lambda)
    model.coef = t[which(t[,1]!=0),]
    print(as.data.frame(model.coef))
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, metricsPlot=metricsPlot ))
  }
  if (method == 'lars'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    # Metrics Plot
    dataPlot = model.caret$results %>%
        gather(key='metric',value='value',-fraction) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=fraction,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    t=coef(model.caret$finalModel,s=model.caret$bestTune$fraction,mode='fraction')
    model.coef = t[which(t!=0)]
    print(model.coef)
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, residHistogram=residHistogram))
  }
}

# https://stackoverflow.com/questions/48265743/linear-model-subset-selection-goodness-of-fit-with-k-fold-cross-validation
# changed slightly since call[[2]] was just returning "formula" without actually returnign the value in formula
predict.regsubsets <- function(object, newdata, id, formula, ...) {
    #form <- as.formula(object$call[[2]])
    mat <- model.matrix(formula, newdata) # adds intercept and expands any interaction terms
    coefi <- coef(object, id = id)
    xvars <- names(coefi)
    return(mat[,xvars]%*%coefi)
}
  
test.model = function(model, test, level=0.95
                      ,draw.limits = FALSE, good = 0.1, ok = 0.15
                      ,method = NULL, subopt = NULL
                      ,id = NULL, formula, feature.names, label.names
                      ,transformation = NULL){
  ## if using caret for glm select equivalent functionality, 
  ## need to pass formula (full is ok as it will select subset of variables from there)
  if (is.null(method)){
    pred = predict(model, newdata=test, interval="confidence", level = level) 
  }
  else if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    pred = predict.regsubsets(model, newdata = test, id = id, formula = formula)
  }
  else if (method == 'glmnet' && subopt == 'LASSO'){
    xtest = as.matrix(test[,feature.names]) 
    pred=as.data.frame(predict(model, xtest))
  }
  else if (method == 'lars'){
    pred=as.data.frame(predict(model, newdata = test))
  }
    
  # Summary of predicted values
  print ("Summary of predicted values: ")
  print(summary(pred[,1]))

  test.mse = mean((test[,label.names]-pred[,1])^2)
  print (paste(method, subopt, "Test MSE:", test.mse, sep=" "))
  
  test.rmse = sqrt(test.mse)
  print (paste(method, subopt, "Test RMSE:", test.rmse, sep=" "))
  
  if(log.pred == TRUE || norm.pred == TRUE){
    # plot transformewd comparison first
    df=data.frame(x=test[,label.names],y=pred[,1])
    ggplot(df,aes(x=x,y=y)) +
      geom_point(color='blue',alpha=0.5,shape=20,size=2) +
      geom_abline(slope=1,intercept=0,color='black',size=1) +
      #scale_y_continuous(limits=c(min(df),max(df)))+
      xlab("Actual (Transformed)")+
      ylab("Predicted (Transformed)")
  }
    
  if (log.pred == FALSE && norm.pred == FALSE){
    x = test[,label.names]
    y = pred[,1]
  }
  if (log.pred == TRUE){
    x = 10^test[,label.names]
    y = 10^pred[,1]
    # x = (test[,label.names])^3
    # y = (pred[,1])^3
  }
  if (norm.pred == TRUE){
    x = predict(transformation, test[,label.names], inverse = TRUE)
    y = predict(transformation, pred[,1], inverse = TRUE)
  }

  test.mse = mean((x-y)^2)
  print (paste(method, subopt, "Test MSE (Org Scale):", test.mse, sep=" "))
  
  test.rmse = sqrt(test.mse)
  print (paste(method, subopt, "Test RMSE (Org Scale):", test.rmse, sep=" "))

  df=data.frame(x,y)
  ggplot(df,aes(x,y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_abline(slope=c(1+good,1-good,1+ok,1-ok)
                ,intercept=rep(0,4),color=c('dark green','dark green','dark red','dark red'),size=1,alpha=0.8) +
    #scale_y_continuous(limits=c(min(df),max(df)))+
    xlab("Actual")+
    ylab("Predicted")
    
 
}

Setup Formulae

n <- names(data.train)
 formula <- as.formula(paste(paste(n[n %in% label.names], collapse = " + ")
                             ," ~", paste(n[!n %in% label.names], collapse = " + "))) 

grand.mean.formula = as.formula(paste(paste(n[n %in% label.names], collapse = " + ")," ~ 1"))

print(formula)
## y3.log ~ PC1 + PC2 + PC3 + PC4 + PC5 + PC6 + PC7 + PC8 + PC9 + 
##     PC10 + PC11 + PC12 + PC13 + PC14 + PC15 + PC16 + PC17 + PC18 + 
##     PC19 + PC20 + PC21 + PC22 + PC23 + PC24 + PC25 + PC26 + PC27 + 
##     PC28 + PC29 + PC30 + PC31 + PC32 + PC33 + PC34 + PC35 + PC36 + 
##     PC37 + PC38 + PC39 + PC40 + PC41 + PC42 + PC43 + PC44 + PC45 + 
##     PC46 + PC47 + PC48 + PC49 + PC50 + PC51 + PC52 + PC53 + PC54 + 
##     PC55 + PC56 + PC57 + PC58 + PC59 + PC60 + PC61 + PC62 + PC63 + 
##     PC64 + PC65 + PC66 + PC67 + PC68 + PC69 + PC70 + PC71 + PC72 + 
##     PC73 + PC74 + PC75 + PC76 + PC77 + PC78 + PC79 + PC80 + PC81 + 
##     PC82 + PC83 + PC84 + PC85 + PC86 + PC87 + PC88 + PC89 + PC90 + 
##     PC91 + PC92 + PC93 + PC94 + PC95 + PC96 + PC97 + PC98 + PC99 + 
##     PC100 + PC101 + PC102 + PC103 + PC104 + PC105 + PC106 + PC107 + 
##     PC108 + PC109 + PC110 + PC111 + PC112 + PC113 + PC114 + PC115 + 
##     PC116 + PC117 + PC118 + PC119 + PC120 + PC121 + PC122 + PC123 + 
##     PC124 + PC125 + PC126 + PC127 + PC128 + PC129 + PC130 + PC131 + 
##     PC132 + PC133 + PC134 + PC135 + PC136 + PC137 + PC138 + PC139 + 
##     PC140 + PC141 + PC142 + PC143 + PC144 + PC145 + PC146 + PC147 + 
##     PC148 + PC149 + PC150 + PC151 + PC152 + PC153 + PC154 + PC155 + 
##     PC156 + PC157 + PC158 + PC159 + PC160 + PC161 + PC162 + PC163 + 
##     PC164
print(grand.mean.formula)
## y3.log ~ 1
# Update feature.names because we may have transformed some features
feature.names = n[!n %in% label.names]

Scatter plots with all PCAs and the output variable (y3.log)

  d = gather(dplyr::select_at(data.pca,c(feature.names,output.var.tr)),key=target,value=value,-!!output.var.tr)
  ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
    geom_point(color='light blue',alpha=0.5) + 
    geom_smooth() + 
    facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

#chart.Correlation(select(data,-JobName),  pch=21)
# https://stackoverflow.com/questions/27034655/how-to-use-dplyrarrangedesc-when-using-a-string-as-column-name
t=as.data.frame(round(cor(dplyr::select(data.pca,-one_of(output.var.tr))
                          ,select_at(data,output.var.tr)),4))  %>%
  rownames_to_column(var='variable') %>% arrange(-!!sym(output.var.tr))
message("Top Positive")
## Top Positive
kable(head(arrange(t,desc(!!sym(output.var.tr))),20))
variable y3.log
PC121 0.0381
PC87 0.0319
PC137 0.0309
PC68 0.0287
PC149 0.0266
PC40 0.0237
PC49 0.0228
PC56 0.0219
PC125 0.0218
PC4 0.0216
PC43 0.0196
PC67 0.0194
PC85 0.0194
PC42 0.0189
PC156 0.0171
PC96 0.0165
PC83 0.0161
PC90 0.0160
PC133 0.0156
PC153 0.0142
message("Top Negative")
## Top Negative
kable(head(arrange(t,!!sym(output.var.tr)),20))  
variable y3.log
PC128 -0.0287
PC76 -0.0251
PC12 -0.0219
PC94 -0.0215
PC54 -0.0213
PC160 -0.0210
PC164 -0.0204
PC70 -0.0200
PC13 -0.0197
PC63 -0.0196
PC115 -0.0196
PC8 -0.0190
PC14 -0.0184
PC124 -0.0181
PC139 -0.0179
PC143 -0.0171
PC81 -0.0161
PC98 -0.0159
PC9 -0.0156
PC145 -0.0153

Full Model

Train

model.full = lm(formula , data.train)
summary(model.full)
## 
## Call:
## lm(formula = formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.085666 -0.022260 -0.005973  0.017026  0.187630 
## 
## Coefficients:
##               Estimate Std. Error  t value Pr(>|t|)    
## (Intercept)  2.097e+00  4.248e-04 4936.182  < 2e-16 ***
## PC1         -4.990e-04  3.722e-05  -13.407  < 2e-16 ***
## PC2         -9.427e-04  3.756e-05  -25.099  < 2e-16 ***
## PC3         -4.374e-04  3.819e-05  -11.451  < 2e-16 ***
## PC4         -3.242e-04  3.850e-05   -8.420  < 2e-16 ***
## PC5          1.771e-04  3.927e-05    4.509 6.64e-06 ***
## PC6         -9.229e-05  3.957e-05   -2.332 0.019727 *  
## PC7         -2.087e-04  4.056e-05   -5.146 2.75e-07 ***
## PC8         -4.702e-05  4.134e-05   -1.137 0.255483    
## PC9         -3.963e-05  4.206e-05   -0.942 0.346149    
## PC10        -1.822e-05  4.303e-05   -0.423 0.671956    
## PC11        -5.806e-04  4.599e-05  -12.625  < 2e-16 ***
## PC12        -4.674e-04  4.845e-05   -9.647  < 2e-16 ***
## PC13         2.936e-04  4.939e-05    5.945 2.94e-09 ***
## PC14         2.353e-04  5.110e-05    4.605 4.23e-06 ***
## PC15        -3.852e-05  5.196e-05   -0.741 0.458548    
## PC16         3.066e-04  5.253e-05    5.837 5.61e-09 ***
## PC17        -1.704e-04  5.536e-05   -3.078 0.002096 ** 
## PC18        -3.526e-04  5.747e-05   -6.135 9.12e-10 ***
## PC19         7.502e-05  5.843e-05    1.284 0.199217    
## PC20         4.102e-04  6.384e-05    6.425 1.43e-10 ***
## PC21         9.972e-05  6.641e-05    1.502 0.133225    
## PC22         1.955e-05  1.043e-04    0.187 0.851322    
## PC23         2.280e-04  1.302e-04    1.751 0.079990 .  
## PC24        -9.013e-04  1.502e-04   -5.999 2.11e-09 ***
## PC25         1.823e-04  1.689e-04    1.079 0.280644    
## PC26         4.830e-04  1.718e-04    2.811 0.004958 ** 
## PC27         2.388e-04  1.757e-04    1.359 0.174232    
## PC28         5.524e-05  1.764e-04    0.313 0.754150    
## PC29         5.622e-04  1.938e-04    2.900 0.003744 ** 
## PC30         5.938e-05  1.977e-04    0.300 0.763936    
## PC31        -1.741e-04  2.109e-04   -0.826 0.408981    
## PC32        -7.593e-04  2.149e-04   -3.533 0.000414 ***
## PC33         4.014e-04  2.188e-04    1.834 0.066696 .  
## PC34         1.089e-03  2.313e-04    4.709 2.55e-06 ***
## PC35         5.475e-05  2.500e-04    0.219 0.826681    
## PC36        -5.923e-06  2.521e-04   -0.023 0.981259    
## PC37        -5.339e-04  2.588e-04   -2.063 0.039171 *  
## PC38         2.825e-04  2.699e-04    1.047 0.295289    
## PC39        -2.365e-04  2.775e-04   -0.852 0.394132    
## PC40        -2.964e-04  2.782e-04   -1.065 0.286786    
## PC41        -2.661e-04  2.854e-04   -0.932 0.351271    
## PC42        -3.905e-05  2.861e-04   -0.136 0.891442    
## PC43        -7.003e-05  2.874e-04   -0.244 0.807533    
## PC44         6.413e-04  2.910e-04    2.204 0.027589 *  
## PC45        -1.101e-04  2.967e-04   -0.371 0.710514    
## PC46         1.938e-04  2.962e-04    0.654 0.513086    
## PC47        -5.010e-04  2.955e-04   -1.695 0.090042 .  
## PC48         3.276e-04  2.952e-04    1.110 0.267183    
## PC49         1.888e-04  3.021e-04    0.625 0.532092    
## PC50        -2.760e-04  3.020e-04   -0.914 0.360817    
## PC51         2.901e-04  3.121e-04    0.930 0.352584    
## PC52         1.969e-04  3.076e-04    0.640 0.522058    
## PC53         1.714e-04  3.071e-04    0.558 0.576736    
## PC54        -3.426e-04  3.121e-04   -1.098 0.272334    
## PC55        -2.023e-04  3.133e-04   -0.646 0.518469    
## PC56        -7.202e-05  3.170e-04   -0.227 0.820282    
## PC57        -1.101e-04  3.161e-04   -0.348 0.727543    
## PC58         4.911e-05  3.204e-04    0.153 0.878193    
## PC59         9.403e-04  3.191e-04    2.946 0.003231 ** 
## PC60        -2.079e-05  3.239e-04   -0.064 0.948814    
## PC61        -2.193e-05  3.211e-04   -0.068 0.945538    
## PC62        -4.283e-04  3.208e-04   -1.335 0.181838    
## PC63        -6.550e-04  3.232e-04   -2.026 0.042777 *  
## PC64        -8.981e-04  3.265e-04   -2.750 0.005970 ** 
## PC65         7.537e-05  3.288e-04    0.229 0.818690    
## PC66        -3.764e-04  3.284e-04   -1.146 0.251736    
## PC67         3.905e-04  3.299e-04    1.184 0.236651    
## PC68         8.195e-04  3.299e-04    2.484 0.013032 *  
## PC69         4.461e-04  3.278e-04    1.361 0.173560    
## PC70        -1.399e-04  3.333e-04   -0.420 0.674811    
## PC71         3.652e-04  3.346e-04    1.092 0.275058    
## PC72        -3.997e-05  3.386e-04   -0.118 0.906026    
## PC73         3.541e-04  3.369e-04    1.051 0.293200    
## PC74        -4.960e-04  3.386e-04   -1.465 0.143032    
## PC75        -8.673e-04  3.343e-04   -2.595 0.009492 ** 
## PC76        -3.092e-04  3.419e-04   -0.904 0.365817    
## PC77         4.001e-04  3.401e-04    1.176 0.239511    
## PC78         1.956e-04  3.423e-04    0.572 0.567670    
## PC79         6.753e-04  3.476e-04    1.943 0.052111 .  
## PC80        -3.711e-04  3.463e-04   -1.071 0.284017    
## PC81         8.750e-04  3.462e-04    2.528 0.011514 *  
## PC82         2.785e-04  3.489e-04    0.798 0.424863    
## PC83        -7.346e-04  3.491e-04   -2.104 0.035421 *  
## PC84         5.812e-04  3.540e-04    1.642 0.100637    
## PC85         1.135e-03  3.530e-04    3.214 0.001315 ** 
## PC86        -3.229e-04  3.544e-04   -0.911 0.362383    
## PC87         1.860e-03  3.536e-04    5.260 1.50e-07 ***
## PC88        -9.439e-04  3.590e-04   -2.629 0.008593 ** 
## PC89        -7.274e-04  3.612e-04   -2.014 0.044099 *  
## PC90        -9.620e-04  3.600e-04   -2.672 0.007564 ** 
## PC91         8.689e-05  3.622e-04    0.240 0.810438    
## PC92        -3.097e-04  3.613e-04   -0.857 0.391407    
## PC93         3.876e-04  3.611e-04    1.073 0.283142    
## PC94        -6.079e-04  3.643e-04   -1.669 0.095228 .  
## PC95         7.706e-05  3.631e-04    0.212 0.831930    
## PC96        -4.607e-04  3.664e-04   -1.257 0.208631    
## PC97        -3.611e-04  3.673e-04   -0.983 0.325543    
## PC98        -5.811e-04  3.659e-04   -1.588 0.112280    
## PC99        -3.775e-04  3.681e-04   -1.025 0.305213    
## PC100        2.517e-04  3.675e-04    0.685 0.493413    
## PC101       -2.724e-05  3.664e-04   -0.074 0.940738    
## PC102       -7.676e-04  3.670e-04   -2.092 0.036504 *  
## PC103        5.305e-04  3.684e-04    1.440 0.149878    
## PC104       -9.005e-04  3.688e-04   -2.442 0.014655 *  
## PC105        9.531e-04  3.697e-04    2.578 0.009951 ** 
## PC106        7.088e-04  3.722e-04    1.904 0.056898 .  
## PC107        1.653e-04  3.734e-04    0.443 0.658013    
## PC108        2.479e-04  3.714e-04    0.667 0.504551    
## PC109        1.320e-04  3.732e-04    0.354 0.723550    
## PC110       -5.208e-04  3.736e-04   -1.394 0.163354    
## PC111       -5.527e-04  3.726e-04   -1.483 0.138013    
## PC112       -8.322e-05  3.728e-04   -0.223 0.823380    
## PC113        2.552e-04  3.753e-04    0.680 0.496555    
## PC114       -6.728e-04  3.736e-04   -1.801 0.071748 .  
## PC115       -1.572e-03  3.795e-04   -4.142 3.49e-05 ***
## PC116       -8.203e-05  3.779e-04   -0.217 0.828161    
## PC117        7.989e-06  3.808e-04    0.021 0.983264    
## PC118        3.522e-04  3.785e-04    0.930 0.352180    
## PC119       -8.767e-04  3.799e-04   -2.308 0.021049 *  
## PC120        3.442e-04  3.786e-04    0.909 0.363321    
## PC121       -4.492e-04  3.814e-04   -1.178 0.238937    
## PC122        6.535e-04  3.815e-04    1.713 0.086745 .  
## PC123       -6.955e-04  3.801e-04   -1.830 0.067308 .  
## PC124        4.768e-04  3.810e-04    1.252 0.210742    
## PC125        4.908e-04  3.852e-04    1.274 0.202656    
## PC126        1.784e-05  3.814e-04    0.047 0.962708    
## PC127        6.673e-04  3.824e-04    1.745 0.081032 .  
## PC128       -9.621e-04  3.838e-04   -2.507 0.012212 *  
## PC129        1.134e-04  3.814e-04    0.297 0.766122    
## PC130        2.236e-04  3.844e-04    0.582 0.560839    
## PC131       -1.130e-03  3.868e-04   -2.923 0.003486 ** 
## PC132        2.470e-04  3.849e-04    0.642 0.521096    
## PC133       -7.617e-05  3.872e-04   -0.197 0.844055    
## PC134        7.093e-04  3.869e-04    1.833 0.066849 .  
## PC135        4.051e-04  3.885e-04    1.043 0.297093    
## PC136        5.726e-04  3.847e-04    1.488 0.136706    
## PC137       -2.995e-04  3.859e-04   -0.776 0.437735    
## PC138        3.868e-04  3.883e-04    0.996 0.319193    
## PC139       -9.158e-04  3.896e-04   -2.351 0.018771 *  
## PC140       -4.054e-04  3.918e-04   -1.035 0.300777    
## PC141       -2.103e-05  3.912e-04   -0.054 0.957134    
## PC142        6.761e-05  3.890e-04    0.174 0.862017    
## PC143        3.884e-04  3.921e-04    0.991 0.321968    
## PC144        7.588e-04  3.907e-04    1.942 0.052190 .  
## PC145        3.897e-04  3.926e-04    0.993 0.320894    
## PC146        7.296e-04  3.931e-04    1.856 0.063512 .  
## PC147       -2.689e-04  3.960e-04   -0.679 0.497076    
## PC148       -5.673e-04  3.954e-04   -1.435 0.151395    
## PC149       -5.301e-05  3.953e-04   -0.134 0.893344    
## PC150        4.800e-04  3.947e-04    1.216 0.224022    
## PC151        5.832e-04  3.970e-04    1.469 0.141852    
## PC152       -5.386e-04  3.933e-04   -1.370 0.170840    
## PC153        8.863e-04  3.952e-04    2.243 0.024958 *  
## PC154       -1.036e-03  3.942e-04   -2.629 0.008582 ** 
## PC155        9.819e-04  3.968e-04    2.475 0.013372 *  
## PC156        5.547e-04  4.002e-04    1.386 0.165752    
## PC157        1.107e-04  3.987e-04    0.278 0.781183    
## PC158        5.985e-06  3.976e-04    0.015 0.987990    
## PC159        1.540e-03  3.952e-04    3.897 9.86e-05 ***
## PC160       -2.228e-04  3.987e-04   -0.559 0.576332    
## PC161        1.258e-04  4.011e-04    0.314 0.753725    
## PC162       -1.400e-03  3.982e-04   -3.516 0.000442 ***
## PC163        6.611e-04  4.018e-04    1.645 0.099956 .  
## PC164        4.038e-04  4.041e-04    0.999 0.317665    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03162 on 5419 degrees of freedom
## Multiple R-squared:  0.2656, Adjusted R-squared:  0.2434 
## F-statistic: 11.95 on 164 and 5419 DF,  p-value: < 2.2e-16
cd.full = plot.diagnostics(model=model.full, train=data.train)

## [1] "Number of data points that have Cook's D > 4/n: 278"
## [1] "Number of data points that have Cook's D > 1: 0"

Test

test.model(model.full, data.test
             ,method = NULL , subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.012   2.088   2.100   2.097   2.110   2.135 
## [1] "  Test MSE: 0.000991150303789864"
## [1] "  Test RMSE: 0.0314825396655013"
## [1] "  Test MSE (Org Scale): 89.3211195242851"
## [1] "  Test RMSE (Org Scale): 9.4509851086691"

Variable Selection

Basic: http://www.stat.columbia.edu/~martin/W2024/R10.pdf Cross Validation + Other Metrics: http://www.sthda.com/english/articles/37-model-selection-essentials-in-r/154-stepwise-regression-essentials-in-r/

Forward Selection with CV

Train

if (algo.forward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   , data = data.train
                                   , method = "leapForward"
                                   , feature.names = feature.names)
  model.forward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 128 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.03472786 0.0875358 0.02693488 0.001061457 0.01850564 0.0006198120
## 2       2 0.03432460 0.1083901 0.02663880 0.001132797 0.01623561 0.0006639239
## 3       3 0.03382946 0.1348557 0.02626974 0.001208496 0.02748852 0.0008560645
## 4       4 0.03346048 0.1536847 0.02596620 0.001299979 0.03210633 0.0009556136
## 5       5 0.03320182 0.1666182 0.02575696 0.001323985 0.03584527 0.0009502264
## 6       6 0.03301559 0.1760576 0.02560187 0.001332234 0.03751422 0.0009523620
## 7       7 0.03301624 0.1761698 0.02562721 0.001335393 0.03785243 0.0009677117
## 8       8 0.03296990 0.1783994 0.02558746 0.001346942 0.03704331 0.0009257019
## 9       9 0.03287814 0.1829320 0.02551140 0.001356576 0.03677055 0.0008862474
## 10     10 0.03278345 0.1876595 0.02543918 0.001359130 0.03655512 0.0008332271
## 11     11 0.03268174 0.1926185 0.02536312 0.001335263 0.03507154 0.0008276001
## 12     12 0.03253215 0.1998258 0.02525020 0.001349551 0.03526951 0.0008867198
## 13     13 0.03245572 0.2035639 0.02521198 0.001389036 0.03650244 0.0009305479
## 14     14 0.03244631 0.2040401 0.02520276 0.001367673 0.03668919 0.0009118742
## 15     15 0.03240234 0.2061430 0.02517818 0.001364078 0.03711829 0.0008930805
## 16     16 0.03233103 0.2096294 0.02511629 0.001345336 0.03744722 0.0008766231
## 17     17 0.03228304 0.2118675 0.02507836 0.001321496 0.03565761 0.0008792641
## 18     18 0.03224579 0.2137794 0.02504090 0.001324247 0.03512276 0.0008644901
## 19     19 0.03223778 0.2142949 0.02504489 0.001371764 0.03522280 0.0009118315
## 20     20 0.03225900 0.2134128 0.02505024 0.001392853 0.03628137 0.0009265227
## 21     21 0.03225984 0.2133994 0.02505656 0.001418306 0.03686905 0.0009380158
## 22     22 0.03225483 0.2136048 0.02504975 0.001423029 0.03688121 0.0009358668
## 23     23 0.03223728 0.2145096 0.02503852 0.001395590 0.03655494 0.0009290173
## 24     24 0.03222293 0.2151330 0.02504058 0.001387110 0.03599535 0.0009286362
## 25     25 0.03223001 0.2148688 0.02503160 0.001393508 0.03576115 0.0009158367
## 26     26 0.03224018 0.2144694 0.02506042 0.001403107 0.03662796 0.0009210134
## 27     27 0.03230391 0.2116293 0.02512815 0.001391499 0.03640708 0.0009015287
## 28     28 0.03231252 0.2111081 0.02512483 0.001360361 0.03429119 0.0008667704
## 29     29 0.03231213 0.2112150 0.02511799 0.001368343 0.03356237 0.0008617138
## 30     30 0.03229978 0.2118183 0.02511427 0.001368913 0.03314928 0.0008575340
## 31     31 0.03231809 0.2109579 0.02512858 0.001375225 0.03311262 0.0008699165
## 32     32 0.03228285 0.2125278 0.02510698 0.001380127 0.03334608 0.0008777023
## 33     33 0.03228394 0.2125305 0.02510694 0.001391326 0.03364957 0.0008957619
## 34     34 0.03228920 0.2123143 0.02510663 0.001407002 0.03427545 0.0009347894
## 35     35 0.03230975 0.2114299 0.02511794 0.001388391 0.03384686 0.0009227280
## 36     36 0.03231350 0.2112871 0.02514065 0.001363555 0.03272151 0.0009090861
## 37     37 0.03230773 0.2116327 0.02513575 0.001371689 0.03344695 0.0009246265
## 38     38 0.03227340 0.2133739 0.02509772 0.001409556 0.03434243 0.0009382950
## 39     39 0.03225123 0.2143758 0.02509063 0.001390021 0.03313816 0.0009162745
## 40     40 0.03226568 0.2137858 0.02511080 0.001396184 0.03341902 0.0009157467
## 41     41 0.03226115 0.2140337 0.02510319 0.001383636 0.03276791 0.0009161375
## 42     42 0.03224718 0.2146750 0.02509158 0.001373413 0.03149069 0.0009103765
## 43     43 0.03225279 0.2145169 0.02510559 0.001383390 0.03193877 0.0009355992
## 44     44 0.03226071 0.2141930 0.02510978 0.001365292 0.03186572 0.0009245905
## 45     45 0.03227103 0.2137277 0.02510361 0.001343984 0.03078506 0.0008973057
## 46     46 0.03226329 0.2141583 0.02509354 0.001349835 0.03091374 0.0009060144
## 47     47 0.03224825 0.2149717 0.02507693 0.001381468 0.03124203 0.0009103212
## 48     48 0.03224030 0.2154032 0.02506404 0.001371514 0.03102525 0.0009122756
## 49     49 0.03224092 0.2153988 0.02506774 0.001383636 0.03137152 0.0009229711
## 50     50 0.03223536 0.2156435 0.02507311 0.001386541 0.03187797 0.0009344750
## 51     51 0.03221723 0.2164997 0.02505882 0.001379601 0.03137269 0.0009396023
## 52     52 0.03221855 0.2163972 0.02506696 0.001370384 0.03053686 0.0009211003
## 53     53 0.03222420 0.2163134 0.02507088 0.001403175 0.03186734 0.0009512459
## 54     54 0.03222994 0.2161084 0.02508507 0.001410654 0.03188335 0.0009518977
## 55     55 0.03222206 0.2165037 0.02507913 0.001410706 0.03143984 0.0009549478
## 56     56 0.03223218 0.2160596 0.02508843 0.001389879 0.03142599 0.0009304736
## 57     57 0.03223685 0.2158818 0.02509162 0.001401075 0.03175593 0.0009393829
## 58     58 0.03222222 0.2165210 0.02508305 0.001393727 0.03143963 0.0009442923
## 59     59 0.03221799 0.2166564 0.02507939 0.001372242 0.03013502 0.0009172113
## 60     60 0.03221669 0.2167642 0.02507295 0.001374715 0.03000050 0.0009196600
## 61     61 0.03221661 0.2167867 0.02507159 0.001381614 0.03013881 0.0009139894
## 62     62 0.03222103 0.2166722 0.02506765 0.001375603 0.03067475 0.0009077495
## 63     63 0.03222210 0.2166116 0.02506447 0.001371666 0.03089435 0.0009035116
## 64     64 0.03223130 0.2162275 0.02507557 0.001380680 0.03136932 0.0009093219
## 65     65 0.03222903 0.2162698 0.02507441 0.001379088 0.03106330 0.0009147433
## 66     66 0.03222641 0.2164138 0.02507278 0.001387152 0.03118630 0.0009232260
## 67     67 0.03223382 0.2161306 0.02507895 0.001398228 0.03135307 0.0009324173
## 68     68 0.03223264 0.2161592 0.02507292 0.001399229 0.03113678 0.0009264410
## 69     69 0.03223728 0.2159839 0.02507449 0.001392739 0.03058663 0.0009243847
## 70     70 0.03224757 0.2155429 0.02508652 0.001377461 0.02997916 0.0009099068
## 71     71 0.03225153 0.2153364 0.02508926 0.001352171 0.02842819 0.0008933890
## 72     72 0.03225051 0.2153966 0.02509251 0.001350439 0.02822789 0.0008987010
## 73     73 0.03225855 0.2150502 0.02509567 0.001351337 0.02820232 0.0009032594
## 74     74 0.03226159 0.2149529 0.02510132 0.001356721 0.02851973 0.0009080979
## 75     75 0.03225516 0.2152672 0.02508647 0.001351262 0.02834770 0.0008958181
## 76     76 0.03225668 0.2152519 0.02509350 0.001347825 0.02796251 0.0008841565
## 77     77 0.03225032 0.2155653 0.02508864 0.001345113 0.02779111 0.0008791602
## 78     78 0.03224405 0.2158242 0.02508998 0.001347392 0.02774745 0.0008887112
## 79     79 0.03224591 0.2157353 0.02508892 0.001346878 0.02778231 0.0008909226
## 80     80 0.03225200 0.2154778 0.02508832 0.001335942 0.02734292 0.0008748025
## 81     81 0.03225884 0.2151564 0.02509153 0.001324637 0.02718162 0.0008606443
## 82     82 0.03223664 0.2161244 0.02508151 0.001321020 0.02672184 0.0008515260
## 83     83 0.03223795 0.2160876 0.02507995 0.001324246 0.02634207 0.0008562950
## 84     84 0.03223261 0.2163689 0.02507210 0.001328825 0.02651001 0.0008510944
## 85     85 0.03222706 0.2166883 0.02506941 0.001336182 0.02681688 0.0008585041
## 86     86 0.03223055 0.2165903 0.02507377 0.001340014 0.02685902 0.0008620817
## 87     87 0.03223617 0.2163166 0.02508194 0.001345909 0.02712090 0.0008660507
## 88     88 0.03223365 0.2164177 0.02508435 0.001340215 0.02702086 0.0008705548
## 89     89 0.03224057 0.2161300 0.02509227 0.001333299 0.02682158 0.0008550341
## 90     90 0.03223284 0.2165345 0.02508732 0.001325548 0.02651749 0.0008445745
## 91     91 0.03222693 0.2168368 0.02508116 0.001332084 0.02680572 0.0008459608
## 92     92 0.03222574 0.2169262 0.02508015 0.001328645 0.02682913 0.0008395214
## 93     93 0.03221636 0.2173616 0.02506876 0.001323183 0.02667344 0.0008264123
## 94     94 0.03220485 0.2178925 0.02506897 0.001319980 0.02651286 0.0008231077
## 95     95 0.03220880 0.2177585 0.02506687 0.001307583 0.02618915 0.0008193251
## 96     96 0.03221366 0.2176087 0.02506833 0.001313508 0.02625488 0.0008252369
## 97     97 0.03221272 0.2176376 0.02506001 0.001309865 0.02604679 0.0008213999
## 98     98 0.03221363 0.2175647 0.02506078 0.001300335 0.02579463 0.0008143892
## 99     99 0.03221161 0.2176594 0.02505901 0.001291401 0.02556742 0.0008118306
## 100   100 0.03220873 0.2177996 0.02505367 0.001293550 0.02558046 0.0008145242
## 101   101 0.03220823 0.2178479 0.02505088 0.001294739 0.02580925 0.0008167034
## 102   102 0.03220142 0.2181472 0.02504322 0.001291011 0.02581538 0.0008207095
## 103   103 0.03219753 0.2183034 0.02504210 0.001281037 0.02570136 0.0008096457
## 104   104 0.03218424 0.2188754 0.02503180 0.001278422 0.02548358 0.0008085554
## 105   105 0.03218079 0.2190303 0.02503498 0.001280441 0.02557842 0.0008131766
## 106   106 0.03217366 0.2193602 0.02502311 0.001277812 0.02569043 0.0008053023
## 107   107 0.03216809 0.2196252 0.02501704 0.001272877 0.02545116 0.0007951466
## 108   108 0.03216808 0.2196173 0.02501264 0.001263216 0.02492189 0.0007931747
## 109   109 0.03217150 0.2195033 0.02501180 0.001264676 0.02482536 0.0007919822
## 110   110 0.03217206 0.2195015 0.02501112 0.001259099 0.02459067 0.0007822882
## 111   111 0.03217058 0.2195809 0.02500716 0.001258084 0.02467756 0.0007838684
## 112   112 0.03217499 0.2193574 0.02501202 0.001263313 0.02446752 0.0007904589
## 113   113 0.03216557 0.2197871 0.02500514 0.001272334 0.02442241 0.0007981316
## 114   114 0.03215465 0.2202602 0.02500504 0.001271621 0.02442841 0.0007996775
## 115   115 0.03215367 0.2203304 0.02500592 0.001272817 0.02438410 0.0007983731
## 116   116 0.03215459 0.2202785 0.02500770 0.001282036 0.02473138 0.0008021585
## 117   117 0.03215325 0.2203364 0.02500540 0.001289368 0.02477652 0.0008065251
## 118   118 0.03215319 0.2203658 0.02500312 0.001294660 0.02496730 0.0008111401
## 119   119 0.03214747 0.2206024 0.02500386 0.001295849 0.02485580 0.0008146161
## 120   120 0.03214932 0.2205358 0.02500884 0.001294564 0.02489271 0.0008095389
## 121   121 0.03214243 0.2208365 0.02500319 0.001292526 0.02479949 0.0008098072
## 122   122 0.03214673 0.2206589 0.02500628 0.001287106 0.02480052 0.0008030332
## 123   123 0.03214132 0.2208758 0.02499794 0.001289169 0.02496264 0.0008045627
## 124   124 0.03213914 0.2209386 0.02499243 0.001284027 0.02479153 0.0008034610
## 125   125 0.03213497 0.2211229 0.02499164 0.001282096 0.02472662 0.0008024946
## 126   126 0.03213254 0.2212350 0.02499147 0.001284816 0.02460441 0.0008020864
## 127   127 0.03213014 0.2213476 0.02499222 0.001286401 0.02466616 0.0008042794
## 128   128 0.03212743 0.2214716 0.02498858 0.001283676 0.02460828 0.0008032679
## 129   129 0.03213257 0.2212367 0.02499516 0.001283804 0.02476711 0.0008022912
## 130   130 0.03214341 0.2207428 0.02500496 0.001282971 0.02465160 0.0008011337
## 131   131 0.03214361 0.2207392 0.02500632 0.001281660 0.02448111 0.0007980568
## 132   132 0.03214713 0.2205711 0.02501125 0.001288916 0.02459372 0.0008063781
## 133   133 0.03214766 0.2205533 0.02500977 0.001291750 0.02465277 0.0008087701
## 134   134 0.03214920 0.2204811 0.02501350 0.001290373 0.02458310 0.0008093175
## 135   135 0.03214700 0.2205805 0.02501083 0.001289549 0.02470230 0.0008083055
## 136   136 0.03214999 0.2204607 0.02501303 0.001292264 0.02475503 0.0008076937
## 137   137 0.03214770 0.2205711 0.02501255 0.001287982 0.02478240 0.0008057101
## 138   138 0.03215044 0.2204692 0.02501635 0.001293295 0.02506123 0.0008099356
## 139   139 0.03215155 0.2204304 0.02501591 0.001295962 0.02521199 0.0008106315
## 140   140 0.03215290 0.2203795 0.02501696 0.001295249 0.02518937 0.0008124483
## 141   141 0.03215340 0.2203547 0.02501605 0.001293884 0.02524891 0.0008131197
## 142   142 0.03215253 0.2203784 0.02501605 0.001289949 0.02505562 0.0008129935
## 143   143 0.03215481 0.2202736 0.02501811 0.001289495 0.02500553 0.0008140239
## 144   144 0.03215505 0.2202574 0.02501788 0.001290488 0.02505785 0.0008160406
## 145   145 0.03215514 0.2202591 0.02501690 0.001290631 0.02509733 0.0008173932
## 146   146 0.03215442 0.2202897 0.02501649 0.001289807 0.02503932 0.0008162902
## 147   147 0.03215308 0.2203552 0.02501437 0.001291392 0.02504550 0.0008164048
## 148   148 0.03215565 0.2202454 0.02501659 0.001290369 0.02505835 0.0008146290
## 149   149 0.03215570 0.2202458 0.02501654 0.001287929 0.02512666 0.0008120514
## 150   150 0.03215597 0.2202318 0.02501680 0.001288054 0.02512420 0.0008109010
## 151   151 0.03215572 0.2202409 0.02501723 0.001287029 0.02506555 0.0008095030
## 152   152 0.03215683 0.2201922 0.02501802 0.001287265 0.02509710 0.0008107259
## 153   153 0.03215690 0.2201956 0.02501849 0.001286687 0.02504495 0.0008093906
## 154   154 0.03215749 0.2201687 0.02501924 0.001286333 0.02501196 0.0008097988
## 155   155 0.03215804 0.2201430 0.02502004 0.001285163 0.02498089 0.0008088150
## 156   156 0.03215702 0.2201853 0.02501986 0.001284499 0.02498216 0.0008079843
## 157   157 0.03215790 0.2201461 0.02502066 0.001284381 0.02494398 0.0008078590
## 158   158 0.03215845 0.2201212 0.02502071 0.001284343 0.02496558 0.0008072354
## 159   159 0.03215958 0.2200704 0.02502139 0.001283451 0.02492590 0.0008067470
## 160   160 0.03216045 0.2200326 0.02502190 0.001283779 0.02494987 0.0008069686
## 161   161 0.03216010 0.2200481 0.02502137 0.001283490 0.02494464 0.0008067311
## 162   162 0.03216047 0.2200324 0.02502148 0.001283635 0.02496082 0.0008069643
## 163   163 0.03216089 0.2200129 0.02502164 0.001283697 0.02495919 0.0008070398
## 164   164 0.03216074 0.2200205 0.02502151 0.001283672 0.02495766 0.0008069270
## [1] "Best Model"
##     nvmax
## 128   128

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.096666e+00  2.095837e+00  2.097495e+00
## PC1         -4.994662e-04 -5.720959e-04 -4.268365e-04
## PC2         -9.425799e-04 -1.015893e-03 -8.692666e-04
## PC3         -4.380802e-04 -5.126305e-04 -3.635300e-04
## PC4         -3.246725e-04 -3.998521e-04 -2.494930e-04
## PC5          1.770479e-04  1.003783e-04  2.537174e-04
## PC6         -9.268306e-05 -1.699501e-04 -1.541603e-05
## PC7         -2.083528e-04 -2.875599e-04 -1.291456e-04
## PC8         -4.679908e-05 -1.275351e-04  3.393696e-05
## PC9         -3.953864e-05 -1.216944e-04  4.261715e-05
## PC11        -5.805753e-04 -6.703796e-04 -4.907710e-04
## PC12        -4.672340e-04 -5.618740e-04 -3.725940e-04
## PC13         2.932873e-04  1.968537e-04  3.897208e-04
## PC14         2.350430e-04  1.352257e-04  3.348602e-04
## PC15        -3.860895e-05 -1.400584e-04  6.284055e-05
## PC16         3.066878e-04  2.040967e-04  4.092789e-04
## PC17        -1.706994e-04 -2.788217e-04 -6.257711e-05
## PC18        -3.514516e-04 -4.636688e-04 -2.392344e-04
## PC19         7.473153e-05 -3.939631e-05  1.888594e-04
## PC20         4.099943e-04  2.852873e-04  5.347013e-04
## PC21         1.001202e-04 -2.953939e-05  2.297798e-04
## PC23         2.266011e-04 -2.756836e-05  4.807705e-04
## PC24        -9.051545e-04 -1.198530e-03 -6.117786e-04
## PC25         1.842789e-04 -1.456391e-04  5.141968e-04
## PC26         4.815861e-04  1.461418e-04  8.170304e-04
## PC27         2.351783e-04 -1.078854e-04  5.782421e-04
## PC29         5.606089e-04  1.821783e-04  9.390395e-04
## PC31        -1.757874e-04 -5.876790e-04  2.361042e-04
## PC32        -7.541658e-04 -1.173771e-03 -3.345610e-04
## PC33         4.016172e-04 -2.574890e-05  8.289833e-04
## PC34         1.088889e-03  6.372229e-04  1.540554e-03
## PC37        -5.342118e-04 -1.039450e-03 -2.897305e-05
## PC38         2.857014e-04 -2.412384e-04  8.126412e-04
## PC39        -2.378230e-04 -7.793169e-04  3.036709e-04
## PC40        -2.950903e-04 -8.379358e-04  2.477551e-04
## PC41        -2.650625e-04 -8.223851e-04  2.922602e-04
## PC44         6.332264e-04  6.503779e-05  1.201415e-03
## PC46         1.925968e-04 -3.853919e-04  7.705856e-04
## PC47        -4.971330e-04 -1.073975e-03  7.970891e-05
## PC48         3.294782e-04 -2.470524e-04  9.060088e-04
## PC49         1.935276e-04 -3.962304e-04  7.832857e-04
## PC50        -2.741322e-04 -8.636318e-04  3.153674e-04
## PC51         2.897821e-04 -3.196233e-04  8.991875e-04
## PC52         1.946909e-04 -4.059983e-04  7.953802e-04
## PC54        -3.441852e-04 -9.535152e-04  2.651449e-04
## PC55        -2.032386e-04 -8.151689e-04  4.086917e-04
## PC59         9.442434e-04  3.209480e-04  1.567539e-03
## PC62        -4.282697e-04 -1.054580e-03  1.980405e-04
## PC63        -6.543528e-04 -1.285391e-03 -2.331480e-05
## PC64        -8.919693e-04 -1.528981e-03 -2.549579e-04
## PC66        -3.803545e-04 -1.021367e-03  2.606580e-04
## PC67         3.984130e-04 -2.457885e-04  1.042614e-03
## PC68         8.215906e-04  1.772576e-04  1.465924e-03
## PC69         4.473591e-04 -1.928159e-04  1.087534e-03
## PC71         3.657414e-04 -2.877059e-04  1.019189e-03
## PC73         3.476116e-04 -3.102413e-04  1.005465e-03
## PC74        -4.917876e-04 -1.152889e-03  1.693141e-04
## PC75        -8.658594e-04 -1.518681e-03 -2.130382e-04
## PC76        -3.027718e-04 -9.700648e-04  3.645212e-04
## PC77         4.053651e-04 -2.587483e-04  1.069478e-03
## PC78         1.992840e-04 -4.690350e-04  8.676030e-04
## PC79         6.750569e-04 -3.670558e-06  1.353784e-03
## PC80        -3.706308e-04 -1.046872e-03  3.056106e-04
## PC81         8.730353e-04  1.975549e-04  1.548516e-03
## PC82         2.759786e-04 -4.053389e-04  9.572961e-04
## PC83        -7.355074e-04 -1.417390e-03 -5.362511e-05
## PC84         5.743941e-04 -1.168969e-04  1.265685e-03
## PC85         1.133195e-03  4.438385e-04  1.822551e-03
## PC86        -3.231296e-04 -1.015046e-03  3.687873e-04
## PC87         1.857972e-03  1.167287e-03  2.548657e-03
## PC88        -9.407818e-04 -1.641708e-03 -2.398561e-04
## PC89        -7.229648e-04 -1.428169e-03 -1.776063e-05
## PC90        -9.635388e-04 -1.666450e-03 -2.606281e-04
## PC92        -3.077542e-04 -1.013458e-03  3.979496e-04
## PC93         3.966031e-04 -3.085869e-04  1.101793e-03
## PC94        -6.123031e-04 -1.323448e-03  9.884158e-05
## PC96        -4.638033e-04 -1.179106e-03  2.514997e-04
## PC97        -3.634298e-04 -1.080697e-03  3.538378e-04
## PC98        -5.831352e-04 -1.297566e-03  1.312953e-04
## PC99        -3.844065e-04 -1.103209e-03  3.343960e-04
## PC100        2.564773e-04 -4.608356e-04  9.737901e-04
## PC102       -7.596988e-04 -1.476301e-03 -4.309708e-05
## PC103        5.301392e-04 -1.891829e-04  1.249461e-03
## PC104       -9.053738e-04 -1.625828e-03 -1.849198e-04
## PC105        9.459939e-04  2.240066e-04  1.667981e-03
## PC106        7.122575e-04 -1.442924e-05  1.438944e-03
## PC108        2.550146e-04 -4.702286e-04  9.802577e-04
## PC110       -5.205370e-04 -1.250010e-03  2.089362e-04
## PC111       -5.514555e-04 -1.279101e-03  1.761904e-04
## PC113        2.494954e-04 -4.836280e-04  9.826188e-04
## PC114       -6.702114e-04 -1.399873e-03  5.945026e-05
## PC115       -1.570461e-03 -2.311792e-03 -8.291294e-04
## PC118        3.585688e-04 -3.806879e-04  1.097825e-03
## PC119       -8.729105e-04 -1.614819e-03 -1.310015e-04
## PC120        3.556027e-04 -3.837860e-04  1.094992e-03
## PC121       -4.485811e-04 -1.193258e-03  2.960957e-04
## PC122        6.494122e-04 -9.553612e-05  1.394361e-03
## PC123       -6.919314e-04 -1.434092e-03  5.022948e-05
## PC124        4.738274e-04 -2.701192e-04  1.217774e-03
## PC125        4.893899e-04 -2.629255e-04  1.241705e-03
## PC127        6.656917e-04 -8.093207e-05  1.412315e-03
## PC128       -9.681493e-04 -1.717745e-03 -2.185541e-04
## PC130        2.226598e-04 -5.280043e-04  9.733240e-04
## PC131       -1.136646e-03 -1.892049e-03 -3.812424e-04
## PC132        2.494469e-04 -5.020361e-04  1.000930e-03
## PC134        7.107453e-04 -4.503247e-05  1.466523e-03
## PC135        4.033936e-04 -3.551793e-04  1.161967e-03
## PC136        5.749529e-04 -1.764404e-04  1.326346e-03
## PC137       -2.987741e-04 -1.052349e-03  4.548012e-04
## PC138        3.869480e-04 -3.712815e-04  1.145178e-03
## PC139       -9.131188e-04 -1.673983e-03 -1.522550e-04
## PC140       -4.156647e-04 -1.180406e-03  3.490764e-04
## PC143        3.873471e-04 -3.781969e-04  1.152891e-03
## PC144        7.572249e-04 -5.753474e-06  1.520203e-03
## PC145        3.886415e-04 -3.779249e-04  1.155208e-03
## PC146        7.268788e-04 -4.091190e-05  1.494669e-03
## PC147       -2.663643e-04 -1.039845e-03  5.071162e-04
## PC148       -5.600192e-04 -1.332133e-03  2.120950e-04
## PC150        4.775381e-04 -2.930204e-04  1.248096e-03
## PC151        5.822327e-04 -1.930573e-04  1.357523e-03
## PC152       -5.352598e-04 -1.303381e-03  2.328610e-04
## PC153        8.832533e-04  1.113578e-04  1.655149e-03
## PC154       -1.030280e-03 -1.800222e-03 -2.603387e-04
## PC155        9.879412e-04  2.129936e-04  1.762889e-03
## PC156        5.463272e-04 -2.353111e-04  1.327965e-03
## PC159        1.542246e-03  7.709317e-04  2.313561e-03
## PC162       -1.401950e-03 -2.179824e-03 -6.240774e-04
## PC163        6.723307e-04 -1.122665e-04  1.456928e-03
## PC164        4.072634e-04 -3.814469e-04  1.195974e-03

Test

if (algo.forward.caret == TRUE){
    test.model(model=model.forward, test=data.test
             ,method = 'leapForward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.011   2.088   2.100   2.097   2.110   2.135 
## [1] "leapForward  Test MSE: 0.000991528663550522"
## [1] "leapForward  Test RMSE: 0.031488548133417"
## [1] "leapForward  Test MSE (Org Scale): 89.3559009387407"
## [1] "leapForward  Test RMSE (Org Scale): 9.45282502423168"

Backward Elimination with CV

Train

if (algo.backward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapBackward"
                                   ,feature.names =  feature.names)
  model.backward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 128 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.03472786 0.0875358 0.02693488 0.001061457 0.01850564 0.0006198120
## 2       2 0.03432460 0.1083901 0.02663880 0.001132797 0.01623561 0.0006639239
## 3       3 0.03382946 0.1348557 0.02626974 0.001208496 0.02748852 0.0008560645
## 4       4 0.03346048 0.1536847 0.02596620 0.001299979 0.03210633 0.0009556136
## 5       5 0.03320182 0.1666182 0.02575696 0.001323985 0.03584527 0.0009502264
## 6       6 0.03301559 0.1760576 0.02560187 0.001332234 0.03751422 0.0009523620
## 7       7 0.03301624 0.1761698 0.02562721 0.001335393 0.03785243 0.0009677117
## 8       8 0.03296990 0.1783994 0.02558746 0.001346942 0.03704331 0.0009257019
## 9       9 0.03287814 0.1829320 0.02551140 0.001356576 0.03677055 0.0008862474
## 10     10 0.03278345 0.1876595 0.02543918 0.001359130 0.03655512 0.0008332271
## 11     11 0.03268174 0.1926185 0.02536312 0.001335263 0.03507154 0.0008276001
## 12     12 0.03253215 0.1998258 0.02525020 0.001349551 0.03526951 0.0008867198
## 13     13 0.03245572 0.2035639 0.02521198 0.001389036 0.03650244 0.0009305479
## 14     14 0.03244559 0.2040930 0.02520042 0.001368563 0.03673819 0.0009141766
## 15     15 0.03240668 0.2059316 0.02517003 0.001358784 0.03693702 0.0009010627
## 16     16 0.03233103 0.2096294 0.02511629 0.001345336 0.03744722 0.0008766231
## 17     17 0.03228304 0.2118675 0.02507836 0.001321496 0.03565761 0.0008792641
## 18     18 0.03224579 0.2137794 0.02504090 0.001324247 0.03512276 0.0008644901
## 19     19 0.03223778 0.2142949 0.02504489 0.001371764 0.03522280 0.0009118315
## 20     20 0.03225900 0.2134128 0.02505024 0.001392853 0.03628137 0.0009265227
## 21     21 0.03225984 0.2133994 0.02505656 0.001418306 0.03686905 0.0009380158
## 22     22 0.03225483 0.2136048 0.02504975 0.001423029 0.03688121 0.0009358668
## 23     23 0.03223728 0.2145096 0.02503852 0.001395590 0.03655494 0.0009290173
## 24     24 0.03222293 0.2151330 0.02504058 0.001387110 0.03599535 0.0009286362
## 25     25 0.03223955 0.2143852 0.02503578 0.001388714 0.03525752 0.0009256415
## 26     26 0.03225127 0.2137937 0.02505803 0.001392841 0.03528662 0.0009391822
## 27     27 0.03230478 0.2114412 0.02511543 0.001374739 0.03504477 0.0009089633
## 28     28 0.03231252 0.2111081 0.02512483 0.001360361 0.03429119 0.0008667704
## 29     29 0.03231090 0.2112681 0.02511968 0.001365963 0.03355562 0.0008640029
## 30     30 0.03229964 0.2118244 0.02511935 0.001368649 0.03314820 0.0008645148
## 31     31 0.03231195 0.2112430 0.02512057 0.001363347 0.03307288 0.0008593860
## 32     32 0.03228285 0.2125278 0.02510698 0.001380127 0.03334608 0.0008777023
## 33     33 0.03228199 0.2126287 0.02510763 0.001391727 0.03367655 0.0008956465
## 34     34 0.03229478 0.2121431 0.02511947 0.001414980 0.03482179 0.0009383104
## 35     35 0.03230763 0.2115287 0.02511912 0.001386249 0.03365635 0.0009231052
## 36     36 0.03231790 0.2110682 0.02514218 0.001358079 0.03250379 0.0009073367
## 37     37 0.03230719 0.2116561 0.02513643 0.001372360 0.03346998 0.0009238903
## 38     38 0.03227255 0.2134134 0.02509780 0.001410602 0.03438104 0.0009382064
## 39     39 0.03225123 0.2143758 0.02509063 0.001390021 0.03313816 0.0009162745
## 40     40 0.03226568 0.2137858 0.02511080 0.001396184 0.03341902 0.0009157467
## 41     41 0.03226115 0.2140337 0.02510319 0.001383636 0.03276791 0.0009161375
## 42     42 0.03224718 0.2146750 0.02509158 0.001373413 0.03149069 0.0009103765
## 43     43 0.03225382 0.2144627 0.02510299 0.001383821 0.03200384 0.0009336970
## 44     44 0.03225800 0.2143041 0.02510986 0.001364040 0.03172435 0.0009246468
## 45     45 0.03226723 0.2138959 0.02510620 0.001342333 0.03057449 0.0008992983
## 46     46 0.03227014 0.2138208 0.02510468 0.001340406 0.03047724 0.0008914093
## 47     47 0.03225285 0.2147433 0.02508381 0.001375393 0.03096309 0.0009017271
## 48     48 0.03224360 0.2152433 0.02506689 0.001367153 0.03083078 0.0009087694
## 49     49 0.03223889 0.2154361 0.02506941 0.001371442 0.03124355 0.0009129428
## 50     50 0.03222749 0.2159614 0.02507009 0.001382848 0.03215921 0.0009317858
## 51     51 0.03222963 0.2159458 0.02505835 0.001397738 0.03249610 0.0009460428
## 52     52 0.03223383 0.2157654 0.02506939 0.001378619 0.03156287 0.0009268465
## 53     53 0.03222863 0.2161335 0.02506729 0.001395336 0.03140917 0.0009439818
## 54     54 0.03224000 0.2155985 0.02509211 0.001393918 0.03099899 0.0009388780
## 55     55 0.03223418 0.2159090 0.02509074 0.001393878 0.03059475 0.0009394035
## 56     56 0.03223218 0.2160596 0.02508843 0.001389879 0.03142599 0.0009304736
## 57     57 0.03223685 0.2158818 0.02509162 0.001401075 0.03175593 0.0009393829
## 58     58 0.03222222 0.2165210 0.02508305 0.001393727 0.03143963 0.0009442923
## 59     59 0.03221799 0.2166564 0.02507939 0.001372242 0.03013502 0.0009172113
## 60     60 0.03221669 0.2167642 0.02507295 0.001374715 0.03000050 0.0009196600
## 61     61 0.03221677 0.2167911 0.02507291 0.001381743 0.03013083 0.0009142891
## 62     62 0.03222116 0.2166776 0.02506941 0.001375715 0.03066510 0.0009081686
## 63     63 0.03221341 0.2169443 0.02505759 0.001369270 0.03062542 0.0009026209
## 64     64 0.03221405 0.2169562 0.02505627 0.001378549 0.03098307 0.0009133733
## 65     65 0.03220829 0.2172380 0.02506142 0.001383679 0.03128122 0.0009094501
## 66     66 0.03221224 0.2170640 0.02507370 0.001392166 0.03176027 0.0009182890
## 67     67 0.03222483 0.2165418 0.02507240 0.001396771 0.03131751 0.0009263191
## 68     68 0.03223364 0.2161190 0.02507534 0.001385673 0.03068932 0.0009133469
## 69     69 0.03222927 0.2163435 0.02506639 0.001386312 0.03066340 0.0009157411
## 70     70 0.03224415 0.2156983 0.02508324 0.001376796 0.03007614 0.0009073967
## 71     71 0.03224721 0.2155414 0.02508778 0.001353251 0.02907374 0.0008935575
## 72     72 0.03225711 0.2151036 0.02509757 0.001354142 0.02869893 0.0009040082
## 73     73 0.03226097 0.2149462 0.02509788 0.001352647 0.02836661 0.0009055452
## 74     74 0.03226159 0.2149529 0.02510132 0.001356721 0.02851973 0.0009080979
## 75     75 0.03225516 0.2152672 0.02508647 0.001351262 0.02834770 0.0008958181
## 76     76 0.03225668 0.2152519 0.02509350 0.001347825 0.02796251 0.0008841565
## 77     77 0.03224701 0.2157146 0.02508491 0.001348569 0.02783423 0.0008838746
## 78     78 0.03224611 0.2157297 0.02508961 0.001345250 0.02772145 0.0008891721
## 79     79 0.03224718 0.2156710 0.02509066 0.001344065 0.02778413 0.0008879319
## 80     80 0.03224682 0.2156896 0.02508690 0.001330162 0.02721748 0.0008700918
## 81     81 0.03225201 0.2154492 0.02508904 0.001322023 0.02705390 0.0008582655
## 82     82 0.03223664 0.2161244 0.02508151 0.001321020 0.02672184 0.0008515260
## 83     83 0.03223795 0.2160876 0.02507995 0.001324246 0.02634207 0.0008562950
## 84     84 0.03222582 0.2166659 0.02506513 0.001336274 0.02662730 0.0008604205
## 85     85 0.03222020 0.2169892 0.02506240 0.001343909 0.02695894 0.0008683547
## 86     86 0.03223055 0.2165903 0.02507377 0.001340014 0.02685902 0.0008620817
## 87     87 0.03223461 0.2163825 0.02507872 0.001342431 0.02706857 0.0008599609
## 88     88 0.03223209 0.2164836 0.02508119 0.001336719 0.02696744 0.0008644918
## 89     89 0.03223967 0.2161828 0.02509126 0.001332700 0.02680514 0.0008544575
## 90     90 0.03223344 0.2165079 0.02508525 0.001332197 0.02659987 0.0008488020
## 91     91 0.03222412 0.2169634 0.02508246 0.001334837 0.02684114 0.0008473485
## 92     92 0.03222305 0.2170220 0.02507983 0.001324039 0.02674285 0.0008337366
## 93     93 0.03221207 0.2175078 0.02507154 0.001316332 0.02645838 0.0008219883
## 94     94 0.03220971 0.2176728 0.02507283 0.001317391 0.02638679 0.0008254195
## 95     95 0.03220782 0.2177521 0.02506350 0.001305749 0.02614404 0.0008216708
## 96     96 0.03221175 0.2176451 0.02506563 0.001312648 0.02622361 0.0008256128
## 97     97 0.03221257 0.2176173 0.02506029 0.001308958 0.02599153 0.0008213930
## 98     98 0.03221818 0.2173602 0.02506236 0.001299372 0.02585117 0.0008148330
## 99     99 0.03221343 0.2175901 0.02505882 0.001289086 0.02545936 0.0008121033
## 100   100 0.03220681 0.2178967 0.02504888 0.001295972 0.02572929 0.0008214924
## 101   101 0.03220694 0.2179238 0.02504739 0.001296372 0.02592640 0.0008218492
## 102   102 0.03220142 0.2181472 0.02504322 0.001291011 0.02581538 0.0008207095
## 103   103 0.03219913 0.2182395 0.02503982 0.001280353 0.02570214 0.0008091609
## 104   104 0.03218508 0.2188446 0.02502872 0.001278065 0.02548428 0.0008078740
## 105   105 0.03218194 0.2189944 0.02503384 0.001278751 0.02549264 0.0008111746
## 106   106 0.03217566 0.2192707 0.02502626 0.001275268 0.02555473 0.0008007975
## 107   107 0.03216809 0.2196252 0.02501704 0.001272877 0.02545116 0.0007951466
## 108   108 0.03216808 0.2196173 0.02501264 0.001263216 0.02492189 0.0007931747
## 109   109 0.03217144 0.2195066 0.02501127 0.001264540 0.02482304 0.0007909065
## 110   110 0.03217519 0.2193623 0.02501282 0.001266321 0.02469393 0.0007857961
## 111   111 0.03217768 0.2192832 0.02501277 0.001274485 0.02490235 0.0007953831
## 112   112 0.03217897 0.2191993 0.02501609 0.001272554 0.02459325 0.0007988454
## 113   113 0.03216557 0.2197871 0.02500514 0.001272334 0.02442241 0.0007981316
## 114   114 0.03215465 0.2202602 0.02500504 0.001271621 0.02442841 0.0007996775
## 115   115 0.03215142 0.2204364 0.02500535 0.001272517 0.02448294 0.0007979897
## 116   116 0.03215236 0.2203834 0.02500719 0.001281701 0.02482331 0.0008018124
## 117   117 0.03215337 0.2203307 0.02500597 0.001289385 0.02477153 0.0008069133
## 118   118 0.03215319 0.2203658 0.02500312 0.001294660 0.02496730 0.0008111401
## 119   119 0.03214747 0.2206024 0.02500386 0.001295849 0.02485580 0.0008146161
## 120   120 0.03214932 0.2205358 0.02500884 0.001294564 0.02489271 0.0008095389
## 121   121 0.03214435 0.2207464 0.02500319 0.001292875 0.02472779 0.0008098091
## 122   122 0.03214769 0.2206137 0.02500441 0.001287279 0.02476441 0.0008017226
## 123   123 0.03214328 0.2207841 0.02499782 0.001289532 0.02489074 0.0008044744
## 124   124 0.03213914 0.2209386 0.02499243 0.001284027 0.02479153 0.0008034610
## 125   125 0.03213497 0.2211229 0.02499164 0.001282096 0.02472662 0.0008024946
## 126   126 0.03213254 0.2212350 0.02499147 0.001284816 0.02460441 0.0008020864
## 127   127 0.03213014 0.2213476 0.02499222 0.001286401 0.02466616 0.0008042794
## 128   128 0.03212743 0.2214716 0.02498858 0.001283676 0.02460828 0.0008032679
## 129   129 0.03213257 0.2212367 0.02499516 0.001283804 0.02476711 0.0008022912
## 130   130 0.03214341 0.2207428 0.02500496 0.001282971 0.02465160 0.0008011337
## 131   131 0.03214361 0.2207392 0.02500632 0.001281660 0.02448111 0.0007980568
## 132   132 0.03214713 0.2205711 0.02501125 0.001288916 0.02459372 0.0008063781
## 133   133 0.03214766 0.2205533 0.02500977 0.001291750 0.02465277 0.0008087701
## 134   134 0.03214920 0.2204811 0.02501350 0.001290373 0.02458310 0.0008093175
## 135   135 0.03214700 0.2205805 0.02501083 0.001289549 0.02470230 0.0008083055
## 136   136 0.03214999 0.2204607 0.02501303 0.001292264 0.02475503 0.0008076937
## 137   137 0.03214770 0.2205711 0.02501255 0.001287982 0.02478240 0.0008057101
## 138   138 0.03215044 0.2204692 0.02501635 0.001293295 0.02506123 0.0008099356
## 139   139 0.03215155 0.2204304 0.02501591 0.001295962 0.02521199 0.0008106315
## 140   140 0.03215407 0.2203298 0.02501793 0.001295102 0.02522243 0.0008115628
## 141   141 0.03215314 0.2203683 0.02501707 0.001293967 0.02524324 0.0008118590
## 142   142 0.03215253 0.2203784 0.02501605 0.001289949 0.02505562 0.0008129935
## 143   143 0.03215481 0.2202736 0.02501811 0.001289495 0.02500553 0.0008140239
## 144   144 0.03215505 0.2202574 0.02501788 0.001290488 0.02505785 0.0008160406
## 145   145 0.03215514 0.2202591 0.02501690 0.001290631 0.02509733 0.0008173932
## 146   146 0.03215442 0.2202897 0.02501649 0.001289807 0.02503932 0.0008162902
## 147   147 0.03215308 0.2203552 0.02501437 0.001291392 0.02504550 0.0008164048
## 148   148 0.03215565 0.2202454 0.02501659 0.001290369 0.02505835 0.0008146290
## 149   149 0.03215570 0.2202458 0.02501654 0.001287929 0.02512666 0.0008120514
## 150   150 0.03215597 0.2202318 0.02501680 0.001288054 0.02512420 0.0008109010
## 151   151 0.03215572 0.2202409 0.02501723 0.001287029 0.02506555 0.0008095030
## 152   152 0.03215683 0.2201922 0.02501802 0.001287265 0.02509710 0.0008107259
## 153   153 0.03215690 0.2201956 0.02501849 0.001286687 0.02504495 0.0008093906
## 154   154 0.03215749 0.2201687 0.02501924 0.001286333 0.02501196 0.0008097988
## 155   155 0.03215804 0.2201430 0.02502004 0.001285163 0.02498089 0.0008088150
## 156   156 0.03215702 0.2201853 0.02501986 0.001284499 0.02498216 0.0008079843
## 157   157 0.03215790 0.2201461 0.02502066 0.001284381 0.02494398 0.0008078590
## 158   158 0.03215845 0.2201212 0.02502071 0.001284343 0.02496558 0.0008072354
## 159   159 0.03215958 0.2200704 0.02502139 0.001283451 0.02492590 0.0008067470
## 160   160 0.03216045 0.2200326 0.02502190 0.001283779 0.02494987 0.0008069686
## 161   161 0.03216010 0.2200481 0.02502137 0.001283490 0.02494464 0.0008067311
## 162   162 0.03216047 0.2200324 0.02502148 0.001283635 0.02496082 0.0008069643
## 163   163 0.03216089 0.2200129 0.02502164 0.001283697 0.02495919 0.0008070398
## 164   164 0.03216074 0.2200205 0.02502151 0.001283672 0.02495766 0.0008069270
## [1] "Best Model"
##     nvmax
## 128   128

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.096666e+00  2.095837e+00  2.097495e+00
## PC1         -4.994662e-04 -5.720959e-04 -4.268365e-04
## PC2         -9.425799e-04 -1.015893e-03 -8.692666e-04
## PC3         -4.380802e-04 -5.126305e-04 -3.635300e-04
## PC4         -3.246725e-04 -3.998521e-04 -2.494930e-04
## PC5          1.770479e-04  1.003783e-04  2.537174e-04
## PC6         -9.268306e-05 -1.699501e-04 -1.541603e-05
## PC7         -2.083528e-04 -2.875599e-04 -1.291456e-04
## PC8         -4.679908e-05 -1.275351e-04  3.393696e-05
## PC9         -3.953864e-05 -1.216944e-04  4.261715e-05
## PC11        -5.805753e-04 -6.703796e-04 -4.907710e-04
## PC12        -4.672340e-04 -5.618740e-04 -3.725940e-04
## PC13         2.932873e-04  1.968537e-04  3.897208e-04
## PC14         2.350430e-04  1.352257e-04  3.348602e-04
## PC15        -3.860895e-05 -1.400584e-04  6.284055e-05
## PC16         3.066878e-04  2.040967e-04  4.092789e-04
## PC17        -1.706994e-04 -2.788217e-04 -6.257711e-05
## PC18        -3.514516e-04 -4.636688e-04 -2.392344e-04
## PC19         7.473153e-05 -3.939631e-05  1.888594e-04
## PC20         4.099943e-04  2.852873e-04  5.347013e-04
## PC21         1.001202e-04 -2.953939e-05  2.297798e-04
## PC23         2.266011e-04 -2.756836e-05  4.807705e-04
## PC24        -9.051545e-04 -1.198530e-03 -6.117786e-04
## PC25         1.842789e-04 -1.456391e-04  5.141968e-04
## PC26         4.815861e-04  1.461418e-04  8.170304e-04
## PC27         2.351783e-04 -1.078854e-04  5.782421e-04
## PC29         5.606089e-04  1.821783e-04  9.390395e-04
## PC31        -1.757874e-04 -5.876790e-04  2.361042e-04
## PC32        -7.541658e-04 -1.173771e-03 -3.345610e-04
## PC33         4.016172e-04 -2.574890e-05  8.289833e-04
## PC34         1.088889e-03  6.372229e-04  1.540554e-03
## PC37        -5.342118e-04 -1.039450e-03 -2.897305e-05
## PC38         2.857014e-04 -2.412384e-04  8.126412e-04
## PC39        -2.378230e-04 -7.793169e-04  3.036709e-04
## PC40        -2.950903e-04 -8.379358e-04  2.477551e-04
## PC41        -2.650625e-04 -8.223851e-04  2.922602e-04
## PC44         6.332264e-04  6.503779e-05  1.201415e-03
## PC46         1.925968e-04 -3.853919e-04  7.705856e-04
## PC47        -4.971330e-04 -1.073975e-03  7.970891e-05
## PC48         3.294782e-04 -2.470524e-04  9.060088e-04
## PC49         1.935276e-04 -3.962304e-04  7.832857e-04
## PC50        -2.741322e-04 -8.636318e-04  3.153674e-04
## PC51         2.897821e-04 -3.196233e-04  8.991875e-04
## PC52         1.946909e-04 -4.059983e-04  7.953802e-04
## PC54        -3.441852e-04 -9.535152e-04  2.651449e-04
## PC55        -2.032386e-04 -8.151689e-04  4.086917e-04
## PC59         9.442434e-04  3.209480e-04  1.567539e-03
## PC62        -4.282697e-04 -1.054580e-03  1.980405e-04
## PC63        -6.543528e-04 -1.285391e-03 -2.331480e-05
## PC64        -8.919693e-04 -1.528981e-03 -2.549579e-04
## PC66        -3.803545e-04 -1.021367e-03  2.606580e-04
## PC67         3.984130e-04 -2.457885e-04  1.042614e-03
## PC68         8.215906e-04  1.772576e-04  1.465924e-03
## PC69         4.473591e-04 -1.928159e-04  1.087534e-03
## PC71         3.657414e-04 -2.877059e-04  1.019189e-03
## PC73         3.476116e-04 -3.102413e-04  1.005465e-03
## PC74        -4.917876e-04 -1.152889e-03  1.693141e-04
## PC75        -8.658594e-04 -1.518681e-03 -2.130382e-04
## PC76        -3.027718e-04 -9.700648e-04  3.645212e-04
## PC77         4.053651e-04 -2.587483e-04  1.069478e-03
## PC78         1.992840e-04 -4.690350e-04  8.676030e-04
## PC79         6.750569e-04 -3.670558e-06  1.353784e-03
## PC80        -3.706308e-04 -1.046872e-03  3.056106e-04
## PC81         8.730353e-04  1.975549e-04  1.548516e-03
## PC82         2.759786e-04 -4.053389e-04  9.572961e-04
## PC83        -7.355074e-04 -1.417390e-03 -5.362511e-05
## PC84         5.743941e-04 -1.168969e-04  1.265685e-03
## PC85         1.133195e-03  4.438385e-04  1.822551e-03
## PC86        -3.231296e-04 -1.015046e-03  3.687873e-04
## PC87         1.857972e-03  1.167287e-03  2.548657e-03
## PC88        -9.407818e-04 -1.641708e-03 -2.398561e-04
## PC89        -7.229648e-04 -1.428169e-03 -1.776063e-05
## PC90        -9.635388e-04 -1.666450e-03 -2.606281e-04
## PC92        -3.077542e-04 -1.013458e-03  3.979496e-04
## PC93         3.966031e-04 -3.085869e-04  1.101793e-03
## PC94        -6.123031e-04 -1.323448e-03  9.884158e-05
## PC96        -4.638033e-04 -1.179106e-03  2.514997e-04
## PC97        -3.634298e-04 -1.080697e-03  3.538378e-04
## PC98        -5.831352e-04 -1.297566e-03  1.312953e-04
## PC99        -3.844065e-04 -1.103209e-03  3.343960e-04
## PC100        2.564773e-04 -4.608356e-04  9.737901e-04
## PC102       -7.596988e-04 -1.476301e-03 -4.309708e-05
## PC103        5.301392e-04 -1.891829e-04  1.249461e-03
## PC104       -9.053738e-04 -1.625828e-03 -1.849198e-04
## PC105        9.459939e-04  2.240066e-04  1.667981e-03
## PC106        7.122575e-04 -1.442924e-05  1.438944e-03
## PC108        2.550146e-04 -4.702286e-04  9.802577e-04
## PC110       -5.205370e-04 -1.250010e-03  2.089362e-04
## PC111       -5.514555e-04 -1.279101e-03  1.761904e-04
## PC113        2.494954e-04 -4.836280e-04  9.826188e-04
## PC114       -6.702114e-04 -1.399873e-03  5.945026e-05
## PC115       -1.570461e-03 -2.311792e-03 -8.291294e-04
## PC118        3.585688e-04 -3.806879e-04  1.097825e-03
## PC119       -8.729105e-04 -1.614819e-03 -1.310015e-04
## PC120        3.556027e-04 -3.837860e-04  1.094992e-03
## PC121       -4.485811e-04 -1.193258e-03  2.960957e-04
## PC122        6.494122e-04 -9.553612e-05  1.394361e-03
## PC123       -6.919314e-04 -1.434092e-03  5.022948e-05
## PC124        4.738274e-04 -2.701192e-04  1.217774e-03
## PC125        4.893899e-04 -2.629255e-04  1.241705e-03
## PC127        6.656917e-04 -8.093207e-05  1.412315e-03
## PC128       -9.681493e-04 -1.717745e-03 -2.185541e-04
## PC130        2.226598e-04 -5.280043e-04  9.733240e-04
## PC131       -1.136646e-03 -1.892049e-03 -3.812424e-04
## PC132        2.494469e-04 -5.020361e-04  1.000930e-03
## PC134        7.107453e-04 -4.503247e-05  1.466523e-03
## PC135        4.033936e-04 -3.551793e-04  1.161967e-03
## PC136        5.749529e-04 -1.764404e-04  1.326346e-03
## PC137       -2.987741e-04 -1.052349e-03  4.548012e-04
## PC138        3.869480e-04 -3.712815e-04  1.145178e-03
## PC139       -9.131188e-04 -1.673983e-03 -1.522550e-04
## PC140       -4.156647e-04 -1.180406e-03  3.490764e-04
## PC143        3.873471e-04 -3.781969e-04  1.152891e-03
## PC144        7.572249e-04 -5.753474e-06  1.520203e-03
## PC145        3.886415e-04 -3.779249e-04  1.155208e-03
## PC146        7.268788e-04 -4.091190e-05  1.494669e-03
## PC147       -2.663643e-04 -1.039845e-03  5.071162e-04
## PC148       -5.600192e-04 -1.332133e-03  2.120950e-04
## PC150        4.775381e-04 -2.930204e-04  1.248096e-03
## PC151        5.822327e-04 -1.930573e-04  1.357523e-03
## PC152       -5.352598e-04 -1.303381e-03  2.328610e-04
## PC153        8.832533e-04  1.113578e-04  1.655149e-03
## PC154       -1.030280e-03 -1.800222e-03 -2.603387e-04
## PC155        9.879412e-04  2.129936e-04  1.762889e-03
## PC156        5.463272e-04 -2.353111e-04  1.327965e-03
## PC159        1.542246e-03  7.709317e-04  2.313561e-03
## PC162       -1.401950e-03 -2.179824e-03 -6.240774e-04
## PC163        6.723307e-04 -1.122665e-04  1.456928e-03
## PC164        4.072634e-04 -3.814469e-04  1.195974e-03

Test

if (algo.backward.caret == TRUE){
  test.model(model.backward, data.test
             ,method = 'leapBackward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.011   2.088   2.100   2.097   2.110   2.135 
## [1] "leapBackward  Test MSE: 0.000991528663550521"
## [1] "leapBackward  Test RMSE: 0.031488548133417"
## [1] "leapBackward  Test MSE (Org Scale): 89.3559009387407"
## [1] "leapBackward  Test RMSE (Org Scale): 9.45282502423168"

Stepwise Selection with CV

Train

if (algo.stepwise.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapSeq"
                                   ,feature.names = feature.names)
  model.stepwise = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 124 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.03472786 0.0875358 0.02693488 0.001061457 0.01850564 0.0006198120
## 2       2 0.03432460 0.1083901 0.02663880 0.001132797 0.01623561 0.0006639239
## 3       3 0.03381950 0.1354576 0.02626943 0.001207490 0.02812201 0.0008561529
## 4       4 0.03371671 0.1400011 0.02616103 0.001175280 0.02553747 0.0007177831
## 5       5 0.03320182 0.1666182 0.02575696 0.001323985 0.03584527 0.0009502264
## 6       6 0.03366187 0.1428485 0.02612199 0.001199574 0.02643690 0.0007409604
## 7       7 0.03301624 0.1761698 0.02562721 0.001335393 0.03785243 0.0009677117
## 8       8 0.03296990 0.1783994 0.02558746 0.001346942 0.03704331 0.0009257019
## 9       9 0.03287814 0.1829320 0.02551140 0.001356576 0.03677055 0.0008862474
## 10     10 0.03278345 0.1876595 0.02543918 0.001359130 0.03655512 0.0008332271
## 11     11 0.03268174 0.1926185 0.02536312 0.001335263 0.03507154 0.0008276001
## 12     12 0.03253215 0.1998258 0.02525020 0.001349551 0.03526951 0.0008867198
## 13     13 0.03252973 0.1998886 0.02527522 0.001464999 0.03897408 0.0010061012
## 14     14 0.03249943 0.2014364 0.02526179 0.001278920 0.03140575 0.0007976783
## 15     15 0.03240610 0.2056590 0.02514527 0.001358204 0.03743793 0.0008940452
## 16     16 0.03250450 0.2010763 0.02524551 0.001286826 0.03953483 0.0009060775
## 17     17 0.03229531 0.2112959 0.02509286 0.001327139 0.03562518 0.0008811913
## 18     18 0.03229176 0.2115476 0.02505879 0.001349441 0.03518790 0.0008803835
## 19     19 0.03223778 0.2142949 0.02504489 0.001371764 0.03522280 0.0009118315
## 20     20 0.03225900 0.2134128 0.02505024 0.001392853 0.03628137 0.0009265227
## 21     21 0.03230060 0.2114551 0.02508710 0.001414429 0.03789628 0.0009606040
## 22     22 0.03225483 0.2136048 0.02504975 0.001423029 0.03688121 0.0009358668
## 23     23 0.03223728 0.2145096 0.02503852 0.001395590 0.03655494 0.0009290173
## 24     24 0.03222293 0.2151330 0.02504058 0.001387110 0.03599535 0.0009286362
## 25     25 0.03223001 0.2148688 0.02503160 0.001393508 0.03576115 0.0009158367
## 26     26 0.03220896 0.2154488 0.02503766 0.001371542 0.03478436 0.0009138839
## 27     27 0.03230478 0.2114412 0.02511543 0.001374739 0.03504477 0.0009089633
## 28     28 0.03226995 0.2128408 0.02509219 0.001326659 0.03312001 0.0008005138
## 29     29 0.03231513 0.2109202 0.02512707 0.001368084 0.03355449 0.0008622217
## 30     30 0.03228440 0.2127970 0.02512507 0.001394761 0.03525297 0.0008526558
## 31     31 0.03233203 0.2101810 0.02513962 0.001343610 0.03298332 0.0008405545
## 32     32 0.03229833 0.2117652 0.02513227 0.001410672 0.03356360 0.0009126150
## 33     33 0.03228394 0.2125305 0.02510694 0.001391326 0.03364957 0.0008957619
## 34     34 0.03228736 0.2124084 0.02510701 0.001407550 0.03431740 0.0009346640
## 35     35 0.03233000 0.2104103 0.02514800 0.001380777 0.03328631 0.0009174022
## 36     36 0.03227811 0.2123645 0.02510663 0.001327187 0.03092459 0.0008960529
## 37     37 0.03231721 0.2110141 0.02513455 0.001376730 0.03341905 0.0009234672
## 38     38 0.03227072 0.2133136 0.02510010 0.001409603 0.03451903 0.0009404868
## 39     39 0.03222710 0.2156048 0.02502507 0.001355812 0.03316403 0.0008697535
## 40     40 0.03226568 0.2137858 0.02511080 0.001396184 0.03341902 0.0009157467
## 41     41 0.03226115 0.2140337 0.02510319 0.001383636 0.03276791 0.0009161375
## 42     42 0.03224718 0.2146750 0.02509158 0.001373413 0.03149069 0.0009103765
## 43     43 0.03225279 0.2145169 0.02510559 0.001383390 0.03193877 0.0009355992
## 44     44 0.03226071 0.2141930 0.02510978 0.001365292 0.03186572 0.0009245905
## 45     45 0.03226723 0.2138959 0.02510620 0.001342333 0.03057449 0.0008992983
## 46     46 0.03230253 0.2120542 0.02512238 0.001308309 0.02967677 0.0008953525
## 47     47 0.03224825 0.2149717 0.02507693 0.001381468 0.03124203 0.0009103212
## 48     48 0.03224403 0.2150374 0.02504876 0.001391638 0.03162426 0.0009171055
## 49     49 0.03227280 0.2138357 0.02507883 0.001343688 0.02970460 0.0009095567
## 50     50 0.03223751 0.2155493 0.02507678 0.001386026 0.03208745 0.0009322012
## 51     51 0.03228722 0.2129211 0.02509804 0.001319199 0.02983498 0.0008974471
## 52     52 0.03225393 0.2140078 0.02507760 0.001376698 0.03289270 0.0009353513
## 53     53 0.03223154 0.2157774 0.02507395 0.001407203 0.03270860 0.0009575115
## 54     54 0.03222911 0.2158386 0.02507144 0.001412416 0.03186882 0.0009475879
## 55     55 0.03227204 0.2138252 0.02512724 0.001396276 0.03234816 0.0009610069
## 56     56 0.03228798 0.2133170 0.02512033 0.001320157 0.02854438 0.0008910529
## 57     57 0.03223685 0.2158818 0.02509162 0.001401075 0.03175593 0.0009393829
## 58     58 0.03219196 0.2181133 0.02507690 0.001440836 0.03457068 0.0009553004
## 59     59 0.03221799 0.2166564 0.02507939 0.001372242 0.03013502 0.0009172113
## 60     60 0.03221669 0.2167642 0.02507295 0.001374715 0.03000050 0.0009196600
## 61     61 0.03225340 0.2149917 0.02510926 0.001368754 0.03040195 0.0009227780
## 62     62 0.03221129 0.2164783 0.02504541 0.001367416 0.03102646 0.0009054078
## 63     63 0.03222210 0.2166116 0.02506447 0.001371666 0.03089435 0.0009035116
## 64     64 0.03223130 0.2162275 0.02507557 0.001380680 0.03136932 0.0009093219
## 65     65 0.03222717 0.2158354 0.02506739 0.001376079 0.03074262 0.0009102436
## 66     66 0.03220459 0.2165475 0.02504843 0.001378992 0.03151796 0.0009165683
## 67     67 0.03226061 0.2147647 0.02508997 0.001361375 0.02984281 0.0009185034
## 68     68 0.03224399 0.2154627 0.02508587 0.001395886 0.03167679 0.0009261175
## 69     69 0.03222927 0.2163435 0.02506639 0.001386312 0.03066340 0.0009157411
## 70     70 0.03223867 0.2159915 0.02504534 0.001379445 0.03049528 0.0008754413
## 71     71 0.03224229 0.2157628 0.02508307 0.001350448 0.02871918 0.0008885700
## 72     72 0.03225051 0.2153966 0.02509251 0.001350439 0.02822789 0.0008987010
## 73     73 0.03226097 0.2149462 0.02509788 0.001352647 0.02836661 0.0009055452
## 74     74 0.03228380 0.2137342 0.02511879 0.001370527 0.03061685 0.0009279178
## 75     75 0.03225516 0.2152672 0.02508647 0.001351262 0.02834770 0.0008958181
## 76     76 0.03224645 0.2157971 0.02506158 0.001346003 0.02835528 0.0008638423
## 77     77 0.03228916 0.2134331 0.02512612 0.001310121 0.02804879 0.0008394870
## 78     78 0.03224405 0.2158242 0.02508998 0.001347392 0.02774745 0.0008887112
## 79     79 0.03223243 0.2164450 0.02505827 0.001340628 0.02836758 0.0008667642
## 80     80 0.03224829 0.2156470 0.02508529 0.001338391 0.02742150 0.0008779441
## 81     81 0.03225201 0.2154492 0.02508904 0.001322023 0.02705390 0.0008582655
## 82     82 0.03228553 0.2134602 0.02512285 0.001295640 0.02980780 0.0008171127
## 83     83 0.03226327 0.2147610 0.02510441 0.001339374 0.02868908 0.0008821531
## 84     84 0.03224074 0.2159390 0.02508436 0.001347345 0.02690431 0.0008750231
## 85     85 0.03217366 0.2190966 0.02506464 0.001410182 0.03086652 0.0008649159
## 86     86 0.03223055 0.2165903 0.02507377 0.001340014 0.02685902 0.0008620817
## 87     87 0.03225179 0.2154918 0.02508679 0.001327631 0.02635142 0.0008612842
## 88     88 0.03218326 0.2186594 0.02507630 0.001407589 0.03107952 0.0008724131
## 89     89 0.03223601 0.2162578 0.02509859 0.001323130 0.02672256 0.0008675088
## 90     90 0.03222869 0.2163234 0.02508410 0.001322355 0.02690422 0.0008443513
## 91     91 0.03222645 0.2162394 0.02508817 0.001322779 0.02768809 0.0008570497
## 92     92 0.03222131 0.2170589 0.02508740 0.001313760 0.02665403 0.0008446662
## 93     93 0.03222808 0.2167246 0.02509482 0.001325979 0.02800580 0.0008443620
## 94     94 0.03216404 0.2193865 0.02504202 0.001280143 0.02604697 0.0008198894
## 95     95 0.03220966 0.2177126 0.02506536 0.001306625 0.02617064 0.0008214433
## 96     96 0.03221366 0.2176087 0.02506833 0.001313508 0.02625488 0.0008252369
## 97     97 0.03224616 0.2156815 0.02508578 0.001284050 0.02462589 0.0008067351
## 98     98 0.03219379 0.2182939 0.02505424 0.001332201 0.02741835 0.0008270563
## 99     99 0.03222081 0.2166399 0.02506240 0.001265762 0.02499927 0.0008001542
## 100   100 0.03220873 0.2177996 0.02505367 0.001293550 0.02558046 0.0008145242
## 101   101 0.03218440 0.2191067 0.02502038 0.001294499 0.02709500 0.0008071130
## 102   102 0.03220142 0.2181472 0.02504322 0.001291011 0.02581538 0.0008207095
## 103   103 0.03219753 0.2183034 0.02504210 0.001281037 0.02570136 0.0008096457
## 104   104 0.03218424 0.2188754 0.02503180 0.001278422 0.02548358 0.0008085554
## 105   105 0.03215698 0.2199506 0.02501938 0.001312806 0.02711482 0.0008333186
## 106   106 0.03216494 0.2196126 0.02501188 0.001280391 0.02557929 0.0007991229
## 107   107 0.03216809 0.2196252 0.02501704 0.001272877 0.02545116 0.0007951466
## 108   108 0.03216808 0.2196173 0.02501264 0.001263216 0.02492189 0.0007931747
## 109   109 0.03220765 0.2175592 0.02503582 0.001228866 0.02507773 0.0007490572
## 110   110 0.03216281 0.2194696 0.02499774 0.001252452 0.02464754 0.0007822069
## 111   111 0.03217374 0.2194405 0.02500877 0.001265356 0.02477941 0.0007871575
## 112   112 0.03217897 0.2191993 0.02501609 0.001272554 0.02459325 0.0007988454
## 113   113 0.03221059 0.2175370 0.02505194 0.001231376 0.02489747 0.0007695118
## 114   114 0.03215465 0.2202602 0.02500504 0.001271621 0.02442841 0.0007996775
## 115   115 0.03215367 0.2203304 0.02500592 0.001272817 0.02438410 0.0007983731
## 116   116 0.03215443 0.2202206 0.02501805 0.001279327 0.02468808 0.0007913078
## 117   117 0.03215337 0.2203307 0.02500597 0.001289385 0.02477153 0.0008069133
## 118   118 0.03215319 0.2203658 0.02500312 0.001294660 0.02496730 0.0008111401
## 119   119 0.03214747 0.2206024 0.02500386 0.001295849 0.02485580 0.0008146161
## 120   120 0.03212725 0.2216414 0.02498119 0.001292636 0.02601157 0.0007946280
## 121   121 0.03214243 0.2208365 0.02500319 0.001292526 0.02479949 0.0008098072
## 122   122 0.03214673 0.2206589 0.02500628 0.001287106 0.02480052 0.0008030332
## 123   123 0.03214328 0.2207841 0.02499782 0.001289532 0.02489074 0.0008044744
## 124   124 0.03211689 0.2220490 0.02497051 0.001281663 0.02587525 0.0007909712
## 125   125 0.03213497 0.2211229 0.02499164 0.001282096 0.02472662 0.0008024946
## 126   126 0.03213254 0.2212350 0.02499147 0.001284816 0.02460441 0.0008020864
## 127   127 0.03213336 0.2208919 0.02498012 0.001288681 0.02546743 0.0008044800
## 128   128 0.03212743 0.2214716 0.02498858 0.001283676 0.02460828 0.0008032679
## 129   129 0.03213257 0.2212367 0.02499516 0.001283804 0.02476711 0.0008022912
## 130   130 0.03216825 0.2196590 0.02503435 0.001296565 0.02650897 0.0008314409
## 131   131 0.03214361 0.2207392 0.02500632 0.001281660 0.02448111 0.0007980568
## 132   132 0.03214713 0.2205711 0.02501125 0.001288916 0.02459372 0.0008063781
## 133   133 0.03214766 0.2205533 0.02500977 0.001291750 0.02465277 0.0008087701
## 134   134 0.03214943 0.2204043 0.02502909 0.001290122 0.02452057 0.0007947717
## 135   135 0.03216709 0.2195886 0.02502240 0.001269519 0.02461970 0.0007930243
## 136   136 0.03214999 0.2204607 0.02501303 0.001292264 0.02475503 0.0008076937
## 137   137 0.03215591 0.2200745 0.02501927 0.001290962 0.02591330 0.0008179877
## 138   138 0.03215248 0.2201059 0.02500481 0.001294733 0.02568932 0.0008105581
## 139   139 0.03215155 0.2204304 0.02501591 0.001295962 0.02521199 0.0008106315
## 140   140 0.03215958 0.2199987 0.02502902 0.001311941 0.02651818 0.0008377046
## 141   141 0.03215280 0.2203826 0.02501692 0.001293162 0.02523046 0.0008115562
## 142   142 0.03218000 0.2190930 0.02506015 0.001284810 0.02572728 0.0008080327
## 143   143 0.03218644 0.2187451 0.02507195 0.001291572 0.02571845 0.0008262860
## 144   144 0.03215505 0.2202574 0.02501788 0.001290488 0.02505785 0.0008160406
## 145   145 0.03217207 0.2194095 0.02504339 0.001272345 0.02451871 0.0007943683
## 146   146 0.03217032 0.2195361 0.02502441 0.001274138 0.02498885 0.0008059002
## 147   147 0.03220094 0.2180900 0.02506324 0.001263666 0.02363101 0.0007857309
## 148   148 0.03215565 0.2202454 0.02501659 0.001290369 0.02505835 0.0008146290
## 149   149 0.03216245 0.2198834 0.02502571 0.001303736 0.02546684 0.0008313923
## 150   150 0.03215597 0.2202318 0.02501680 0.001288054 0.02512420 0.0008109010
## 151   151 0.03215572 0.2202409 0.02501723 0.001287029 0.02506555 0.0008095030
## 152   152 0.03216535 0.2197636 0.02501903 0.001293670 0.02531494 0.0008113569
## 153   153 0.03216415 0.2198791 0.02501624 0.001274298 0.02521856 0.0007978609
## 154   154 0.03215900 0.2200496 0.02502126 0.001285577 0.02499426 0.0008100083
## 155   155 0.03215397 0.2203284 0.02501973 0.001277795 0.02568308 0.0007918531
## 156   156 0.03215651 0.2201276 0.02502549 0.001284651 0.02494687 0.0008004501
## 157   157 0.03217359 0.2193932 0.02503116 0.001294963 0.02560334 0.0008152898
## 158   158 0.03215957 0.2200378 0.02502870 0.001283079 0.02489780 0.0007995420
## 159   159 0.03216329 0.2198960 0.02503298 0.001281394 0.02483354 0.0007912617
## 160   160 0.03216528 0.2198199 0.02503462 0.001283491 0.02500424 0.0007916376
## 161   161 0.03215893 0.2200704 0.02502906 0.001284812 0.02496307 0.0007993303
## 162   162 0.03215962 0.2200953 0.02501711 0.001283095 0.02519516 0.0008018994
## 163   163 0.03215896 0.2200985 0.02501904 0.001284831 0.02506568 0.0008072135
## 164   164 0.03216074 0.2200205 0.02502151 0.001283672 0.02495766 0.0008069270
## [1] "Best Model"
##     nvmax
## 124   124

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  2.096659e+00  2.095830e+00  2.097487e+00
## PC1         -4.997658e-04 -5.723624e-04 -4.271691e-04
## PC2         -9.419626e-04 -1.015241e-03 -8.686837e-04
## PC3         -4.391224e-04 -5.136222e-04 -3.646225e-04
## PC4         -3.242618e-04 -3.994039e-04 -2.491197e-04
## PC5          1.765051e-04  9.986175e-05  2.531485e-04
## PC6         -9.231281e-05 -1.695287e-04 -1.509692e-05
## PC7         -2.082596e-04 -2.874469e-04 -1.290722e-04
## PC8         -4.693927e-05 -1.276475e-04  3.376897e-05
## PC9         -3.909225e-05 -1.212191e-04  4.303461e-05
## PC11        -5.810165e-04 -6.707886e-04 -4.912445e-04
## PC12        -4.676546e-04 -5.622673e-04 -3.730418e-04
## PC13         2.930867e-04  1.966760e-04  3.894975e-04
## PC14         2.344941e-04  1.347109e-04  3.342773e-04
## PC15        -3.911419e-05 -1.405327e-04  6.230431e-05
## PC16         3.058276e-04  2.032819e-04  4.083733e-04
## PC17        -1.701312e-04 -2.782192e-04 -6.204333e-05
## PC18        -3.519583e-04 -4.641377e-04 -2.397790e-04
## PC19         7.500380e-05 -3.907661e-05  1.890842e-04
## PC20         4.106839e-04  2.860206e-04  5.353471e-04
## PC21         1.003118e-04 -2.931462e-05  2.299383e-04
## PC23         2.287595e-04 -2.523984e-05  4.827588e-04
## PC24        -9.062350e-04 -1.199519e-03 -6.129510e-04
## PC25         1.862813e-04 -1.435207e-04  5.160832e-04
## PC26         4.823554e-04  1.469997e-04  8.177110e-04
## PC27         2.339783e-04 -1.089839e-04  5.769405e-04
## PC29         5.611705e-04  1.828666e-04  9.394743e-04
## PC31        -1.736812e-04 -5.854572e-04  2.380949e-04
## PC32        -7.551321e-04 -1.174630e-03 -3.356339e-04
## PC33         4.006520e-04 -2.659617e-05  8.279002e-04
## PC34         1.089153e-03  6.376159e-04  1.540691e-03
## PC37        -5.373194e-04 -1.042393e-03 -3.224590e-05
## PC38         2.849708e-04 -2.418253e-04  8.117669e-04
## PC39        -2.362448e-04 -7.775214e-04  3.050319e-04
## PC40        -2.997290e-04 -8.423369e-04  2.428789e-04
## PC41        -2.599751e-04 -8.170540e-04  2.971038e-04
## PC44         6.290816e-04  6.109832e-05  1.197065e-03
## PC46         1.968107e-04 -3.809470e-04  7.745684e-04
## PC47        -4.996877e-04 -1.076348e-03  7.697243e-05
## PC48         3.284368e-04 -2.479308e-04  9.048044e-04
## PC50        -2.767511e-04 -8.660549e-04  3.125526e-04
## PC51         2.887351e-04 -3.205127e-04  8.979829e-04
## PC54        -3.444124e-04 -9.535879e-04  2.647630e-04
## PC55        -2.019717e-04 -8.137421e-04  4.097987e-04
## PC59         9.408372e-04  3.177595e-04  1.563915e-03
## PC62        -4.323139e-04 -1.058433e-03  1.938047e-04
## PC63        -6.542496e-04 -1.285126e-03 -2.337289e-05
## PC64        -8.888503e-04 -1.525695e-03 -2.520054e-04
## PC66        -3.863940e-04 -1.027139e-03  2.543513e-04
## PC67         3.974831e-04 -2.465421e-04  1.041508e-03
## PC68         8.225872e-04  1.784373e-04  1.466737e-03
## PC69         4.439242e-04 -1.959698e-04  1.083818e-03
## PC71         3.673069e-04 -2.859701e-04  1.020584e-03
## PC73         3.476716e-04 -3.098426e-04  1.005186e-03
## PC74        -4.957264e-04 -1.156618e-03  1.651655e-04
## PC75        -8.646934e-04 -1.517344e-03 -2.120426e-04
## PC76        -2.949266e-04 -9.618776e-04  3.720243e-04
## PC77         3.958113e-04 -2.678934e-04  1.059516e-03
## PC79         6.774454e-04 -1.028822e-06  1.355920e-03
## PC80        -3.720502e-04 -1.048083e-03  3.039823e-04
## PC81         8.760508e-04  2.007494e-04  1.551352e-03
## PC82         2.760300e-04 -4.050903e-04  9.571503e-04
## PC83        -7.404018e-04 -1.422055e-03 -5.874850e-05
## PC84         5.688482e-04 -1.222141e-04  1.259910e-03
## PC85         1.132068e-03  4.430202e-04  1.821115e-03
## PC86        -3.199491e-04 -1.011677e-03  3.717785e-04
## PC87         1.856071e-03  1.165554e-03  2.546588e-03
## PC88        -9.436457e-04 -1.644344e-03 -2.429477e-04
## PC89        -7.220978e-04 -1.427100e-03 -1.709597e-05
## PC90        -9.642077e-04 -1.666900e-03 -2.615151e-04
## PC92        -3.067944e-04 -1.012283e-03  3.986939e-04
## PC93         3.952580e-04 -3.096411e-04  1.100157e-03
## PC94        -6.082842e-04 -1.319194e-03  1.026254e-04
## PC96        -4.615116e-04 -1.176606e-03  2.535832e-04
## PC97        -3.579184e-04 -1.074938e-03  3.591017e-04
## PC98        -5.814605e-04 -1.295714e-03  1.327928e-04
## PC99        -3.850739e-04 -1.103673e-03  3.335248e-04
## PC100        2.548386e-04 -4.622816e-04  9.719588e-04
## PC102       -7.647606e-04 -1.481113e-03 -4.840848e-05
## PC103        5.365540e-04 -1.825206e-04  1.255629e-03
## PC104       -9.099239e-04 -1.630139e-03 -1.897084e-04
## PC105        9.468319e-04  2.250600e-04  1.668604e-03
## PC106        7.199742e-04 -6.395549e-06  1.446344e-03
## PC108        2.510471e-04 -4.739897e-04  9.760839e-04
## PC110       -5.150037e-04 -1.244192e-03  2.141848e-04
## PC111       -5.564896e-04 -1.283900e-03  1.709204e-04
## PC113        2.520449e-04 -4.808782e-04  9.849680e-04
## PC114       -6.700358e-04 -1.399527e-03  5.945587e-05
## PC115       -1.569273e-03 -2.310368e-03 -8.281787e-04
## PC118        3.622080e-04 -3.767847e-04  1.101201e-03
## PC119       -8.696544e-04 -1.611315e-03 -1.279942e-04
## PC120        3.561252e-04 -3.830543e-04  1.095305e-03
## PC121       -4.511182e-04 -1.195585e-03  2.933490e-04
## PC122        6.495029e-04 -9.519583e-05  1.394202e-03
## PC123       -6.912184e-04 -1.433141e-03  5.070373e-05
## PC124        4.813707e-04 -2.622837e-04  1.225025e-03
## PC125        4.965890e-04 -2.554384e-04  1.248616e-03
## PC127        6.695609e-04 -7.681537e-05  1.415937e-03
## PC128       -9.699995e-04 -1.719401e-03 -2.205976e-04
## PC131       -1.138864e-03 -1.894067e-03 -3.836605e-04
## PC132        2.479453e-04 -5.033553e-04  9.992459e-04
## PC134        7.156726e-04 -3.984502e-05  1.471190e-03
## PC135        4.064518e-04 -3.519045e-04  1.164808e-03
## PC136        5.759509e-04 -1.752275e-04  1.327129e-03
## PC137       -2.976947e-04 -1.051023e-03  4.556341e-04
## PC138        3.849774e-04 -3.729822e-04  1.142937e-03
## PC139       -9.271984e-04 -1.687508e-03 -1.668891e-04
## PC140       -4.131939e-04 -1.177735e-03  3.513470e-04
## PC143        3.846308e-04 -3.807100e-04  1.149972e-03
## PC144        7.586957e-04 -4.079424e-06  1.521471e-03
## PC145        3.843754e-04 -3.818809e-04  1.150632e-03
## PC146        7.268206e-04 -4.075834e-05  1.494400e-03
## PC147       -2.733259e-04 -1.046434e-03  4.997820e-04
## PC148       -5.575318e-04 -1.329454e-03  2.143908e-04
## PC150        4.804003e-04 -2.898563e-04  1.250657e-03
## PC151        5.801210e-04 -1.949755e-04  1.355217e-03
## PC152       -5.376380e-04 -1.305558e-03  2.302819e-04
## PC153        8.797658e-04  1.081048e-04  1.651427e-03
## PC154       -1.031704e-03 -1.801461e-03 -2.619474e-04
## PC155        9.919059e-04  2.172260e-04  1.766586e-03
## PC156        5.465264e-04 -2.348923e-04  1.327945e-03
## PC159        1.541973e-03  7.709181e-04  2.313028e-03
## PC162       -1.399424e-03 -2.177076e-03 -6.217719e-04
## PC163        6.740592e-04 -1.103491e-04  1.458468e-03
## PC164        4.061922e-04 -3.823332e-04  1.194718e-03

Test

if (algo.stepwise.caret == TRUE){
  test.model(model.stepwise, data.test
             ,method = 'leapSeq',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
  
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.011   2.088   2.100   2.097   2.110   2.135 
## [1] "leapSeq  Test MSE: 0.000991647321730291"
## [1] "leapSeq  Test RMSE: 0.031490432225206"
## [1] "leapSeq  Test MSE (Org Scale): 89.371209069609"
## [1] "leapSeq  Test RMSE (Org Scale): 9.45363470151079"

LASSO with CV

Train

if (algo.LASSO.caret == TRUE){
  set.seed(1)
  tune.grid= expand.grid(alpha = 1,lambda = 10^seq(from=-4,to=-2,length=100))
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "glmnet"
                                   ,subopt = 'LASSO'
                                   ,tune.grid = tune.grid
                                   ,feature.names = feature.names)
  model.LASSO.caret = returned$model
}
## Aggregating results
## Selecting tuning parameters
## Fitting alpha = 1, lambda = 0.000231 on full training set
## glmnet 
## 
## 5584 samples
##  164 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   lambda        RMSE        Rsquared    MAE       
##   0.0001000000  0.03206691  0.22273155  0.02495043
##   0.0001047616  0.03206391  0.22282063  0.02494799
##   0.0001097499  0.03206091  0.22291004  0.02494550
##   0.0001149757  0.03205796  0.22299671  0.02494305
##   0.0001204504  0.03205507  0.22308091  0.02494064
##   0.0001261857  0.03205221  0.22316453  0.02493836
##   0.0001321941  0.03204941  0.22324608  0.02493617
##   0.0001384886  0.03204672  0.22332370  0.02493407
##   0.0001450829  0.03204420  0.22339517  0.02493216
##   0.0001519911  0.03204171  0.22346690  0.02493028
##   0.0001592283  0.03203933  0.22353503  0.02492844
##   0.0001668101  0.03203700  0.22360310  0.02492666
##   0.0001747528  0.03203481  0.22366723  0.02492496
##   0.0001830738  0.03203288  0.22372216  0.02492329
##   0.0001917910  0.03203121  0.22376933  0.02492182
##   0.0002009233  0.03202984  0.22380592  0.02492066
##   0.0002104904  0.03202876  0.22383321  0.02491968
##   0.0002205131  0.03202816  0.22384278  0.02491900
##   0.0002310130  0.03202780  0.22384676  0.02491856
##   0.0002420128  0.03202791  0.22383505  0.02491848
##   0.0002535364  0.03202843  0.22381065  0.02491862
##   0.0002656088  0.03202946  0.22376909  0.02491928
##   0.0002782559  0.03203110  0.22370545  0.02492053
##   0.0002915053  0.03203306  0.22363569  0.02492237
##   0.0003053856  0.03203559  0.22354857  0.02492491
##   0.0003199267  0.03203878  0.22344109  0.02492788
##   0.0003351603  0.03204264  0.22331225  0.02493144
##   0.0003511192  0.03204682  0.22318108  0.02493503
##   0.0003678380  0.03205092  0.22306905  0.02493802
##   0.0003853529  0.03205564  0.22294090  0.02494093
##   0.0004037017  0.03206097  0.22279969  0.02494367
##   0.0004229243  0.03206731  0.22262537  0.02494775
##   0.0004430621  0.03207470  0.22241689  0.02495259
##   0.0004641589  0.03208274  0.22219555  0.02495802
##   0.0004862602  0.03209152  0.22195794  0.02496417
##   0.0005094138  0.03210119  0.22169865  0.02497092
##   0.0005336699  0.03211197  0.22140865  0.02497811
##   0.0005590810  0.03212387  0.22108573  0.02498648
##   0.0005857021  0.03213675  0.22073930  0.02499542
##   0.0006135907  0.03215144  0.22032263  0.02500538
##   0.0006428073  0.03216723  0.21987524  0.02501579
##   0.0006734151  0.03218542  0.21932544  0.02502758
##   0.0007054802  0.03220531  0.21871222  0.02504081
##   0.0007390722  0.03222726  0.21801751  0.02505641
##   0.0007742637  0.03225082  0.21726814  0.02507334
##   0.0008111308  0.03227619  0.21645139  0.02509152
##   0.0008497534  0.03230376  0.21554615  0.02511139
##   0.0008902151  0.03233253  0.21460422  0.02513263
##   0.0009326033  0.03236245  0.21363124  0.02515506
##   0.0009770100  0.03239437  0.21257490  0.02517818
##   0.0010235310  0.03242824  0.21144012  0.02520305
##   0.0010722672  0.03246467  0.21019135  0.02523016
##   0.0011233240  0.03250367  0.20882714  0.02525922
##   0.0011768120  0.03254472  0.20737691  0.02528891
##   0.0012328467  0.03258719  0.20587836  0.02531991
##   0.0012915497  0.03263218  0.20425443  0.02535444
##   0.0013530478  0.03268071  0.20245385  0.02539259
##   0.0014174742  0.03273113  0.20058074  0.02543315
##   0.0014849683  0.03278394  0.19861023  0.02547449
##   0.0015556761  0.03283961  0.19651017  0.02551767
##   0.0016297508  0.03289780  0.19429990  0.02556230
##   0.0017073526  0.03296067  0.19182999  0.02560999
##   0.0017886495  0.03302663  0.18919372  0.02565996
##   0.0018738174  0.03309465  0.18645303  0.02571082
##   0.0019630407  0.03316465  0.18360519  0.02576381
##   0.0020565123  0.03323661  0.18064880  0.02581820
##   0.0021544347  0.03331106  0.17754247  0.02587536
##   0.0022570197  0.03338658  0.17439772  0.02593247
##   0.0023644894  0.03346288  0.17122249  0.02598912
##   0.0024770764  0.03353856  0.16811213  0.02604487
##   0.0025950242  0.03361084  0.16525317  0.02609812
##   0.0027185882  0.03368216  0.16253185  0.02615112
##   0.0028480359  0.03375246  0.15996791  0.02620376
##   0.0029836472  0.03382541  0.15730335  0.02625793
##   0.0031257158  0.03390285  0.15437395  0.02631551
##   0.0032745492  0.03398526  0.15108974  0.02637643
##   0.0034304693  0.03407237  0.14743846  0.02644037
##   0.0035938137  0.03416165  0.14364904  0.02650688
##   0.0037649358  0.03425401  0.13960415  0.02657682
##   0.0039442061  0.03435186  0.13505242  0.02664947
##   0.0041320124  0.03445355  0.13006914  0.02672408
##   0.0043287613  0.03455652  0.12489601  0.02679935
##   0.0045348785  0.03466265  0.11929005  0.02687605
##   0.0047508102  0.03477428  0.11293742  0.02695476
##   0.0049770236  0.03489122  0.10573843  0.02703583
##   0.0052140083  0.03499735  0.09942032  0.02710982
##   0.0054622772  0.03509164  0.09407642  0.02717650
##   0.0057223677  0.03517014  0.09042241  0.02723150
##   0.0059948425  0.03523158  0.08865713  0.02727428
##   0.0062802914  0.03528952  0.08765748  0.02731403
##   0.0065793322  0.03534464  0.08753580  0.02735176
##   0.0068926121  0.03540425  0.08753580  0.02739312
##   0.0072208090  0.03546956  0.08753580  0.02743846
##   0.0075646333  0.03554109  0.08753580  0.02748881
##   0.0079248290  0.03561944  0.08753580  0.02754457
##   0.0083021757  0.03570523  0.08753580  0.02760591
##   0.0086974900  0.03579916  0.08753580  0.02767336
##   0.0091116276  0.03590195  0.08753580  0.02774740
##   0.0095454846  0.03601444  0.08753580  0.02782846
##   0.0100000000  0.03613749  0.08753580  0.02791698
## 
## Tuning parameter 'alpha' was held constant at a value of 1
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were alpha = 1 and lambda = 0.000231013.

##    alpha      lambda
## 19     1 0.000231013
##     alpha       lambda       RMSE   Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.0001000000 0.03206691 0.22273155 0.02495043 0.001301184 0.02629425 0.0008268577
## 2       1 0.0001047616 0.03206391 0.22282063 0.02494799 0.001301748 0.02636188 0.0008275958
## 3       1 0.0001097499 0.03206091 0.22291004 0.02494550 0.001302386 0.02643705 0.0008284132
## 4       1 0.0001149757 0.03205796 0.22299671 0.02494305 0.001303004 0.02651346 0.0008292026
## 5       1 0.0001204504 0.03205507 0.22308091 0.02494064 0.001303614 0.02659290 0.0008300075
## 6       1 0.0001261857 0.03205221 0.22316453 0.02493836 0.001304230 0.02667516 0.0008309140
## 7       1 0.0001321941 0.03204941 0.22324608 0.02493617 0.001304883 0.02676276 0.0008319103
## 8       1 0.0001384886 0.03204672 0.22332370 0.02493407 0.001305724 0.02686022 0.0008329435
## 9       1 0.0001450829 0.03204420 0.22339517 0.02493216 0.001306668 0.02697188 0.0008341111
## 10      1 0.0001519911 0.03204171 0.22346690 0.02493028 0.001307587 0.02709014 0.0008352419
## 11      1 0.0001592283 0.03203933 0.22353503 0.02492844 0.001308563 0.02721720 0.0008364414
## 12      1 0.0001668101 0.03203700 0.22360310 0.02492666 0.001309445 0.02734968 0.0008376470
## 13      1 0.0001747528 0.03203481 0.22366723 0.02492496 0.001310361 0.02749060 0.0008389737
## 14      1 0.0001830738 0.03203288 0.22372216 0.02492329 0.001311381 0.02764172 0.0008404477
## 15      1 0.0001917910 0.03203121 0.22376933 0.02492182 0.001312463 0.02780157 0.0008421437
## 16      1 0.0002009233 0.03202984 0.22380592 0.02492066 0.001313664 0.02796884 0.0008440928
## 17      1 0.0002104904 0.03202876 0.22383321 0.02491968 0.001314828 0.02813608 0.0008460493
## 18      1 0.0002205131 0.03202816 0.22384278 0.02491900 0.001316252 0.02831730 0.0008480620
## 19      1 0.0002310130 0.03202780 0.22384676 0.02491856 0.001317648 0.02850711 0.0008502188
## 20      1 0.0002420128 0.03202791 0.22383505 0.02491848 0.001319224 0.02871348 0.0008524270
## 21      1 0.0002535364 0.03202843 0.22381065 0.02491862 0.001320917 0.02893552 0.0008548890
## 22      1 0.0002656088 0.03202946 0.22376909 0.02491928 0.001322487 0.02915692 0.0008572801
## 23      1 0.0002782559 0.03203110 0.22370545 0.02492053 0.001324068 0.02938239 0.0008596569
## 24      1 0.0002915053 0.03203306 0.22363569 0.02492237 0.001325542 0.02961107 0.0008621045
## 25      1 0.0003053856 0.03203559 0.22354857 0.02492491 0.001327144 0.02985294 0.0008644832
## 26      1 0.0003199267 0.03203878 0.22344109 0.02492788 0.001328868 0.03010651 0.0008668954
## 27      1 0.0003351603 0.03204264 0.22331225 0.02493144 0.001330713 0.03037107 0.0008690598
## 28      1 0.0003511192 0.03204682 0.22318108 0.02493503 0.001332808 0.03062660 0.0008713043
## 29      1 0.0003678380 0.03205092 0.22306905 0.02493802 0.001334635 0.03086721 0.0008730631
## 30      1 0.0003853529 0.03205564 0.22294090 0.02494093 0.001335948 0.03110943 0.0008746998
## 31      1 0.0004037017 0.03206097 0.22279969 0.02494367 0.001336935 0.03134211 0.0008756920
## 32      1 0.0004229243 0.03206731 0.22262537 0.02494775 0.001337175 0.03157925 0.0008760306
## 33      1 0.0004430621 0.03207470 0.22241689 0.02495259 0.001337013 0.03182420 0.0008762115
## 34      1 0.0004641589 0.03208274 0.22219555 0.02495802 0.001336551 0.03210970 0.0008765049
## 35      1 0.0004862602 0.03209152 0.22195794 0.02496417 0.001336165 0.03243373 0.0008765803
## 36      1 0.0005094138 0.03210119 0.22169865 0.02497092 0.001335336 0.03278721 0.0008761730
## 37      1 0.0005336699 0.03211197 0.22140865 0.02497811 0.001334457 0.03317385 0.0008755025
## 38      1 0.0005590810 0.03212387 0.22108573 0.02498648 0.001333403 0.03356634 0.0008750911
## 39      1 0.0005857021 0.03213675 0.22073930 0.02499542 0.001332357 0.03396576 0.0008739453
## 40      1 0.0006135907 0.03215144 0.22032263 0.02500538 0.001330777 0.03432230 0.0008725561
## 41      1 0.0006428073 0.03216723 0.21987524 0.02501579 0.001328523 0.03461141 0.0008706940
## 42      1 0.0006734151 0.03218542 0.21932544 0.02502758 0.001326330 0.03486695 0.0008692381
## 43      1 0.0007054802 0.03220531 0.21871222 0.02504081 0.001323669 0.03508322 0.0008681489
## 44      1 0.0007390722 0.03222726 0.21801751 0.02505641 0.001320641 0.03524005 0.0008662818
## 45      1 0.0007742637 0.03225082 0.21726814 0.02507334 0.001316838 0.03533547 0.0008628490
## 46      1 0.0008111308 0.03227619 0.21645139 0.02509152 0.001312329 0.03539944 0.0008579621
## 47      1 0.0008497534 0.03230376 0.21554615 0.02511139 0.001307851 0.03545621 0.0008525260
## 48      1 0.0008902151 0.03233253 0.21460422 0.02513263 0.001303854 0.03555253 0.0008482212
## 49      1 0.0009326033 0.03236245 0.21363124 0.02515506 0.001299750 0.03565113 0.0008436369
## 50      1 0.0009770100 0.03239437 0.21257490 0.02517818 0.001295936 0.03569740 0.0008391248
## 51      1 0.0010235310 0.03242824 0.21144012 0.02520305 0.001292076 0.03567792 0.0008352084
## 52      1 0.0010722672 0.03246467 0.21019135 0.02523016 0.001287740 0.03565971 0.0008313204
## 53      1 0.0011233240 0.03250367 0.20882714 0.02525922 0.001283214 0.03562620 0.0008267793
## 54      1 0.0011768120 0.03254472 0.20737691 0.02528891 0.001279594 0.03560564 0.0008228628
## 55      1 0.0012328467 0.03258719 0.20587836 0.02531991 0.001276106 0.03555858 0.0008189733
## 56      1 0.0012915497 0.03263218 0.20425443 0.02535444 0.001270932 0.03546545 0.0008145193
## 57      1 0.0013530478 0.03268071 0.20245385 0.02539259 0.001265262 0.03533846 0.0008089808
## 58      1 0.0014174742 0.03273113 0.20058074 0.02543315 0.001259478 0.03525494 0.0008049380
## 59      1 0.0014849683 0.03278394 0.19861023 0.02547449 0.001254152 0.03516183 0.0008008046
## 60      1 0.0015556761 0.03283961 0.19651017 0.02551767 0.001249845 0.03509915 0.0007974267
## 61      1 0.0016297508 0.03289780 0.19429990 0.02556230 0.001246188 0.03498435 0.0007938842
## 62      1 0.0017073526 0.03296067 0.19182999 0.02560999 0.001243260 0.03485858 0.0007909743
## 63      1 0.0017886495 0.03302663 0.18919372 0.02565996 0.001239240 0.03458251 0.0007866890
## 64      1 0.0018738174 0.03309465 0.18645303 0.02571082 0.001233825 0.03431504 0.0007820097
## 65      1 0.0019630407 0.03316465 0.18360519 0.02576381 0.001227575 0.03392991 0.0007770585
## 66      1 0.0020565123 0.03323661 0.18064880 0.02581820 0.001220223 0.03360988 0.0007720950
## 67      1 0.0021544347 0.03331106 0.17754247 0.02587536 0.001212511 0.03317658 0.0007660547
## 68      1 0.0022570197 0.03338658 0.17439772 0.02593247 0.001205411 0.03290893 0.0007606235
## 69      1 0.0023644894 0.03346288 0.17122249 0.02598912 0.001198541 0.03254935 0.0007544805
## 70      1 0.0024770764 0.03353856 0.16811213 0.02604487 0.001193000 0.03221190 0.0007480254
## 71      1 0.0025950242 0.03361084 0.16525317 0.02609812 0.001186845 0.03133578 0.0007387651
## 72      1 0.0027185882 0.03368216 0.16253185 0.02615112 0.001181758 0.03059021 0.0007315860
## 73      1 0.0028480359 0.03375246 0.15996791 0.02620376 0.001176745 0.02972760 0.0007245296
## 74      1 0.0029836472 0.03382541 0.15730335 0.02625793 0.001170248 0.02896053 0.0007167329
## 75      1 0.0031257158 0.03390285 0.15437395 0.02631551 0.001164510 0.02816716 0.0007098716
## 76      1 0.0032745492 0.03398526 0.15108974 0.02637643 0.001158704 0.02730925 0.0007031831
## 77      1 0.0034304693 0.03407237 0.14743846 0.02644037 0.001151462 0.02625801 0.0006957113
## 78      1 0.0035938137 0.03416165 0.14364904 0.02650688 0.001143052 0.02539118 0.0006869717
## 79      1 0.0037649358 0.03425401 0.13960415 0.02657682 0.001135647 0.02435175 0.0006767473
## 80      1 0.0039442061 0.03435186 0.13505242 0.02664947 0.001129496 0.02336392 0.0006675005
## 81      1 0.0041320124 0.03445355 0.13006914 0.02672408 0.001121707 0.02214163 0.0006589907
## 82      1 0.0043287613 0.03455652 0.12489601 0.02679935 0.001114111 0.02129832 0.0006495905
## 83      1 0.0045348785 0.03466265 0.11929005 0.02687605 0.001108719 0.02039996 0.0006404807
## 84      1 0.0047508102 0.03477428 0.11293742 0.02695476 0.001105383 0.01972434 0.0006334689
## 85      1 0.0049770236 0.03489122 0.10573843 0.02703583 0.001102942 0.01895416 0.0006272675
## 86      1 0.0052140083 0.03499735 0.09942032 0.02710982 0.001103551 0.01915754 0.0006230842
## 87      1 0.0054622772 0.03509164 0.09407642 0.02717650 0.001106203 0.01873843 0.0006208467
## 88      1 0.0057223677 0.03517014 0.09042241 0.02723150 0.001105430 0.01942431 0.0006173868
## 89      1 0.0059948425 0.03523158 0.08865713 0.02727428 0.001103603 0.01882196 0.0006131887
## 90      1 0.0062802914 0.03528952 0.08765748 0.02731403 0.001104265 0.01873406 0.0006114533
## 91      1 0.0065793322 0.03534464 0.08753580 0.02735176 0.001108800 0.01850564 0.0006119928
## 92      1 0.0068926121 0.03540425 0.08753580 0.02739312 0.001111828 0.01850564 0.0006104488
## 93      1 0.0072208090 0.03546956 0.08753580 0.02743846 0.001115087 0.01850564 0.0006090071
## 94      1 0.0075646333 0.03554109 0.08753580 0.02748881 0.001118598 0.01850564 0.0006084258
## 95      1 0.0079248290 0.03561944 0.08753580 0.02754457 0.001122384 0.01850564 0.0006088559
## 96      1 0.0083021757 0.03570523 0.08753580 0.02760591 0.001126470 0.01850564 0.0006088667
## 97      1 0.0086974900 0.03579916 0.08753580 0.02767336 0.001130884 0.01850564 0.0006102204
## 98      1 0.0091116276 0.03590195 0.08753580 0.02774740 0.001135658 0.01850564 0.0006110478
## 99      1 0.0095454846 0.03601444 0.08753580 0.02782846 0.001140824 0.01850564 0.0006113757
## 100     1 0.0100000000 0.03613749 0.08753580 0.02791698 0.001146421 0.01850564 0.0006121823

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##                model.coef
## (Intercept)  2.096658e+00
## PC1         -4.799507e-04
## PC2         -9.202807e-04
## PC3         -4.188304e-04
## PC4         -3.036556e-04
## PC5          1.545492e-04
## PC6         -7.287966e-05
## PC7         -1.852841e-04
## PC8         -2.371220e-05
## PC9         -1.473350e-05
## PC11        -5.574330e-04
## PC12        -4.432590e-04
## PC13         2.658715e-04
## PC14         2.057163e-04
## PC15        -9.883151e-06
## PC16         2.820791e-04
## PC17        -1.397320e-04
## PC18        -3.223685e-04
## PC19         4.371053e-05
## PC20         3.759103e-04
## PC21         6.562980e-05
## PC23         1.604885e-04
## PC24        -8.127199e-04
## PC25         9.447366e-05
## PC26         3.883762e-04
## PC27         1.433760e-04
## PC29         4.678172e-04
## PC31        -5.083275e-05
## PC32        -6.229101e-04
## PC33         2.752230e-04
## PC34         9.643209e-04
## PC37        -3.961231e-04
## PC38         1.317506e-04
## PC39        -9.073278e-05
## PC40        -1.419130e-04
## PC41        -9.691812e-05
## PC44         4.563543e-04
## PC46         2.482365e-05
## PC47        -3.559654e-04
## PC48         1.462162e-04
## PC49         3.190472e-05
## PC50        -1.005604e-04
## PC51         9.709880e-05
## PC52         4.559111e-05
## PC53         1.685521e-05
## PC54        -1.418605e-04
## PC55        -3.777624e-06
## PC59         7.663104e-04
## PC62        -2.485663e-04
## PC63        -4.533437e-04
## PC64        -6.967440e-04
## PC66        -2.249583e-04
## PC67         2.018743e-04
## PC68         6.493853e-04
## PC69         2.705157e-04
## PC71         1.870183e-04
## PC73         1.520674e-04
## PC74        -3.101111e-04
## PC75        -6.621970e-04
## PC76        -9.430839e-05
## PC77         2.122278e-04
## PC78         2.856317e-06
## PC79         4.491448e-04
## PC80        -2.072996e-04
## PC81         6.794533e-04
## PC82         7.453161e-05
## PC83        -5.500302e-04
## PC84         3.827331e-04
## PC85         9.556405e-04
## PC86        -1.472490e-04
## PC87         1.686909e-03
## PC88        -7.844941e-04
## PC89        -5.265443e-04
## PC90        -7.423377e-04
## PC92        -9.137152e-05
## PC93         1.950928e-04
## PC94        -3.926244e-04
## PC96        -2.532271e-04
## PC97        -1.626748e-04
## PC98        -3.879666e-04
## PC99        -1.842468e-04
## PC100        4.332014e-05
## PC102       -5.730935e-04
## PC103        3.211222e-04
## PC104       -7.007930e-04
## PC105        7.593030e-04
## PC106        5.196788e-04
## PC107        1.849664e-06
## PC108        3.840142e-05
## PC110       -3.029180e-04
## PC111       -3.680055e-04
## PC113        9.948791e-06
## PC114       -4.921065e-04
## PC115       -1.352155e-03
## PC118        1.366918e-04
## PC119       -6.466373e-04
## PC120        1.570469e-04
## PC121       -2.330221e-04
## PC122        4.255873e-04
## PC123       -5.002018e-04
## PC124        2.656131e-04
## PC125        3.067373e-04
## PC127        4.568164e-04
## PC128       -7.472134e-04
## PC130        3.661383e-05
## PC131       -9.552720e-04
## PC132        4.168643e-05
## PC134        4.966364e-04
## PC135        1.731627e-04
## PC136        3.573157e-04
## PC137       -6.419016e-05
## PC138        1.527863e-04
## PC139       -6.879850e-04
## PC140       -2.205019e-04
## PC143        1.841870e-04
## PC144        5.571087e-04
## PC145        1.711036e-04
## PC146        5.228375e-04
## PC147       -2.538072e-05
## PC148       -3.229516e-04
## PC150        2.512381e-04
## PC151        3.735442e-04
## PC152       -3.215586e-04
## PC153        6.591491e-04
## PC154       -8.289342e-04
## PC155        7.768053e-04
## PC156        3.520305e-04
## PC159        1.306405e-03
## PC160       -4.432069e-06
## PC162       -1.180210e-03
## PC163        4.714407e-04
## PC164        1.654573e-04

Test

if (algo.LASSO.caret == TRUE){
  test.model(model.LASSO.caret, data.test
             ,method = 'glmnet',subopt = "LASSO"
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.020   2.089   2.100   2.097   2.108   2.131 
## [1] "glmnet LASSO Test MSE: 0.000979874292188973"
## [1] "glmnet LASSO Test RMSE: 0.0313029438262438"
## [1] "glmnet LASSO Test MSE (Org Scale): 88.393945839481"
## [1] "glmnet LASSO Test RMSE (Org Scale): 9.40180545637278"

LARS with CV

Train

if (algo.LARS.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "lars"
                                   ,subopt = 'NULL'
                                   ,feature.names = feature.names)
  model.LARS.caret = returned$model
}
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, : There were missing values in resampled
## performance measures.
## Aggregating results
## Selecting tuning parameters
## Fitting fraction = 0.758 on full training set
## Least Angle Regression 
## 
## 5584 samples
##  164 predictor
## 
## Pre-processing: centered (164), scaled (164) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   fraction    RMSE        Rsquared    MAE       
##   0.00000000  0.03633304         NaN  0.02805647
##   0.01010101  0.03594473  0.08753580  0.02777660
##   0.02020202  0.03560752  0.08753580  0.02753502
##   0.03030303  0.03532291  0.08753813  0.02733629
##   0.04040404  0.03510328  0.09327323  0.02718199
##   0.05050505  0.03489864  0.10534976  0.02703808
##   0.06060606  0.03470472  0.11726401  0.02690229
##   0.07070707  0.03452341  0.12673322  0.02677245
##   0.08080808  0.03435738  0.13505587  0.02665108
##   0.09090909  0.03420002  0.14229513  0.02653332
##   0.10101010  0.03405726  0.14826356  0.02642617
##   0.11111111  0.03392161  0.15385061  0.02632679
##   0.12121212  0.03379411  0.15863145  0.02623270
##   0.13131313  0.03368381  0.16259280  0.02615037
##   0.14141414  0.03358645  0.16629656  0.02607975
##   0.15151515  0.03349410  0.16991903  0.02601197
##   0.16161616  0.03340519  0.17358212  0.02594618
##   0.17171717  0.03331591  0.17736025  0.02587875
##   0.18181818  0.03323082  0.18094079  0.02581407
##   0.19191919  0.03315114  0.18422514  0.02575318
##   0.20202020  0.03307613  0.18723946  0.02569655
##   0.21212121  0.03300423  0.19013139  0.02564268
##   0.22222222  0.03293600  0.19283179  0.02559089
##   0.23232323  0.03287153  0.19532237  0.02554160
##   0.24242424  0.03281121  0.19758730  0.02549441
##   0.25252525  0.03275397  0.19971506  0.02544903
##   0.26262626  0.03270103  0.20169065  0.02540642
##   0.27272727  0.03265212  0.20350437  0.02536807
##   0.28282828  0.03260754  0.20512489  0.02533427
##   0.29292929  0.03256683  0.20656808  0.02530395
##   0.30303030  0.03253043  0.20782590  0.02527742
##   0.31313131  0.03249573  0.20903467  0.02525172
##   0.32323232  0.03246350  0.21014862  0.02522821
##   0.33333333  0.03243296  0.21120360  0.02520585
##   0.34343434  0.03240473  0.21215413  0.02518466
##   0.35353535  0.03237881  0.21300860  0.02516534
##   0.36363636  0.03235510  0.21376891  0.02514773
##   0.37373737  0.03233246  0.21450822  0.02513077
##   0.38383838  0.03230980  0.21527687  0.02511413
##   0.39393939  0.03228875  0.21597715  0.02509883
##   0.40404040  0.03226976  0.21658862  0.02508529
##   0.41414141  0.03225217  0.21714198  0.02507293
##   0.42424242  0.03223510  0.21768955  0.02506081
##   0.43434343  0.03221920  0.21819476  0.02505001
##   0.44444444  0.03220441  0.21866153  0.02503992
##   0.45454545  0.03219072  0.21907952  0.02503050
##   0.46464646  0.03217769  0.21947493  0.02502175
##   0.47474747  0.03216548  0.21983918  0.02501350
##   0.48484848  0.03215402  0.22017821  0.02500597
##   0.49494949  0.03214360  0.22047675  0.02499900
##   0.50505051  0.03213402  0.22074372  0.02499248
##   0.51515152  0.03212522  0.22097889  0.02498662
##   0.52525253  0.03211655  0.22121862  0.02498089
##   0.53535354  0.03210843  0.22144371  0.02497555
##   0.54545455  0.03210123  0.22163516  0.02497072
##   0.55555556  0.03209450  0.22181533  0.02496616
##   0.56565657  0.03208819  0.22198360  0.02496166
##   0.57575758  0.03208202  0.22215620  0.02495731
##   0.58585859  0.03207606  0.22232689  0.02495324
##   0.59595960  0.03207043  0.22248950  0.02494952
##   0.60606061  0.03206507  0.22264797  0.02494593
##   0.61616162  0.03206034  0.22278358  0.02494309
##   0.62626263  0.03205640  0.22288952  0.02494105
##   0.63636364  0.03205281  0.22298714  0.02493898
##   0.64646465  0.03204961  0.22307404  0.02493681
##   0.65656566  0.03204642  0.22316937  0.02493416
##   0.66666667  0.03204309  0.22327910  0.02493131
##   0.67676768  0.03203971  0.22339752  0.02492833
##   0.68686869  0.03203696  0.22349168  0.02492575
##   0.69696970  0.03203467  0.22357032  0.02492349
##   0.70707071  0.03203261  0.22364505  0.02492163
##   0.71717172  0.03203065  0.22372206  0.02491995
##   0.72727273  0.03202937  0.22377170  0.02491899
##   0.73737374  0.03202859  0.22380316  0.02491863
##   0.74747475  0.03202807  0.22382793  0.02491860
##   0.75757576  0.03202800  0.22383937  0.02491879
##   0.76767677  0.03202821  0.22384436  0.02491923
##   0.77777778  0.03202897  0.22382999  0.02491997
##   0.78787879  0.03203025  0.22379842  0.02492103
##   0.79797980  0.03203201  0.22375022  0.02492263
##   0.80808081  0.03203433  0.22368313  0.02492461
##   0.81818182  0.03203711  0.22360268  0.02492686
##   0.82828283  0.03204024  0.22351281  0.02492927
##   0.83838384  0.03204366  0.22341542  0.02493189
##   0.84848485  0.03204747  0.22330785  0.02493483
##   0.85858586  0.03205187  0.22318104  0.02493825
##   0.86868687  0.03205681  0.22303697  0.02494224
##   0.87878788  0.03206223  0.22287746  0.02494673
##   0.88888889  0.03206830  0.22269606  0.02495166
##   0.89898990  0.03207460  0.22251145  0.02495672
##   0.90909091  0.03208134  0.22231410  0.02496229
##   0.91919192  0.03208857  0.22210179  0.02496797
##   0.92929293  0.03209631  0.22187439  0.02497387
##   0.93939394  0.03210468  0.22162633  0.02498010
##   0.94949495  0.03211309  0.22138215  0.02498636
##   0.95959596  0.03212168  0.22113625  0.02499285
##   0.96969697  0.03213062  0.22088171  0.02499956
##   0.97979798  0.03214005  0.22061426  0.02500645
##   0.98989899  0.03215022  0.22032139  0.02501379
##   1.00000000  0.03216074  0.22002046  0.02502151
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was fraction = 0.7575758.

##     fraction
## 76 0.7575758
## Warning: Removed 1 rows containing missing values (geom_point).

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##           PC1           PC2           PC3           PC4           PC5           PC6           PC7           PC8 
## -5.485657e-03 -1.041466e-02 -4.666547e-03 -3.352334e-03  1.674324e-03 -7.843493e-04 -1.941968e-03 -2.456482e-04 
##           PC9          PC11          PC12          PC13          PC14          PC15          PC16          PC17 
## -1.508113e-04 -5.148967e-03 -3.885866e-03  2.289433e-03  1.710877e-03 -8.274828e-05  2.281916e-03 -1.073565e-03 
##          PC18          PC19          PC20          PC21          PC23          PC24          PC25          PC26 
## -2.383663e-03  3.195022e-04  2.501730e-03  4.212262e-04  5.264985e-04 -2.299298e-03  2.392920e-04  9.617553e-04 
##          PC27          PC29          PC31          PC32          PC33          PC34          PC37          PC38 
##  3.486263e-04  1.026691e-03 -1.043353e-04 -1.233282e-03  5.358367e-04  1.772480e-03 -6.520341e-04  2.093846e-04 
##          PC39          PC40          PC41          PC44          PC46          PC47          PC48          PC49 
## -1.409753e-04 -2.188864e-04 -1.465466e-04  6.691759e-04  3.759333e-05 -5.136293e-04  2.124405e-04  4.668183e-05 
##          PC50          PC51          PC52          PC53          PC54          PC55          PC59          PC62 
## -1.434519e-04  1.343929e-04  6.471637e-05  2.515771e-05 -1.954908e-04 -4.805508e-06  1.022173e-03 -3.312307e-04 
##          PC63          PC64          PC66          PC67          PC68          PC69          PC71          PC73 
## -5.981931e-04 -9.097403e-04 -2.928747e-04  2.619159e-04  8.380380e-04  3.522387e-04  2.392924e-04  1.936656e-04 
##          PC74          PC75          PC76          PC77          PC79          PC80          PC81          PC82 
## -3.908251e-04 -8.434583e-04 -1.192546e-04  2.671789e-04  5.511900e-04 -2.558768e-04  8.357282e-04  9.270868e-05 
##          PC83          PC84          PC85          PC86          PC87          PC88          PC89          PC90 
## -6.710509e-04  4.609955e-04  1.151090e-03 -1.781077e-04  2.026792e-03 -9.300281e-04 -6.212049e-04 -8.781471e-04 
##          PC92          PC93          PC94          PC96          PC97          PC98          PC99         PC100 
## -1.094686e-04  2.312894e-04 -4.600767e-04 -2.954271e-04 -1.898988e-04 -4.524109e-04 -2.143553e-04  5.203105e-05 
##         PC102         PC103         PC104         PC105         PC106         PC108         PC110         PC111 
## -6.647983e-04  3.721583e-04 -8.084112e-04  8.735407e-04  5.949578e-04  4.589616e-05 -3.463711e-04 -4.209386e-04 
##         PC113         PC114         PC115         PC118         PC119         PC120         PC121         PC122 
##  1.352546e-05 -5.608260e-04 -1.514517e-03  1.553867e-04 -7.250585e-04  1.778366e-04 -2.614094e-04  4.757536e-04 
##         PC123         PC124         PC125         PC127         PC128         PC130         PC131         PC132 
## -5.603038e-04  2.979124e-04  3.399717e-04  5.095509e-04 -8.283895e-04  4.218912e-05 -1.050207e-03  4.782348e-05 
##         PC134         PC135         PC136         PC137         PC138         PC139         PC140         PC143 
##  5.470400e-04  1.914542e-04  3.963328e-04 -7.273543e-05  1.690818e-04 -7.524366e-04 -2.408588e-04  2.013255e-04 
##         PC144         PC145         PC146         PC147         PC148         PC150         PC151         PC152 
##  6.076572e-04  1.869404e-04  5.662762e-04 -2.930924e-05 -3.488870e-04  2.723191e-04  4.011508e-04 -3.491758e-04 
##         PC153         PC154         PC155         PC156         PC159         PC160         PC162         PC163 
##  7.098434e-04 -8.940905e-04  8.330511e-04  3.751425e-04  1.406579e-03 -5.149710e-06 -1.260241e-03  4.999147e-04 
##         PC164 
##  1.759616e-04

Test

if (algo.LARS.caret == TRUE){
  test.model(model.LARS.caret, data.test
             ,method = 'lars',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.020   2.089   2.100   2.097   2.109   2.131 
## [1] "lars  Test MSE: 0.000979913265796265"
## [1] "lars  Test RMSE: 0.0313035663430904"
## [1] "lars  Test MSE (Org Scale): 88.3971956781316"
## [1] "lars  Test RMSE (Org Scale): 9.40197828534674"

Session Info

sessionInfo()
## R version 3.5.1 (2018-07-02)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 17134)
## 
## Matrix products: default
## 
## locale:
## [1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
## [4] LC_NUMERIC=C                           LC_TIME=English_United States.1252    
## 
## attached base packages:
## [1] parallel  stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] knitr_1.20                 htmltools_0.3.6            reshape2_1.4.3             lars_1.2                  
##  [5] doParallel_1.0.14          iterators_1.0.10           caret_6.0-81               leaps_3.0                 
##  [9] ggforce_0.1.3              rlist_0.4.6.1              car_3.0-2                  carData_3.0-2             
## [13] bestNormalize_1.3.0        scales_1.0.0               onewaytests_2.0            caTools_1.17.1.1          
## [17] mosaic_1.5.0               mosaicData_0.17.0          ggformula_0.9.1            ggstance_0.3.1            
## [21] lattice_0.20-35            DT_0.5                     ggiraphExtra_0.2.9         ggiraph_0.6.0             
## [25] investr_1.4.0              glmnet_2.0-16              foreach_1.4.4              Matrix_1.2-14             
## [29] MASS_7.3-50                PerformanceAnalytics_1.5.2 xts_0.11-2                 zoo_1.8-4                 
## [33] forcats_0.3.0              stringr_1.3.1              dplyr_0.8.0.1              purrr_0.2.5               
## [37] readr_1.3.1                tidyr_0.8.2                tibble_2.1.1               ggplot2_3.1.0             
## [41] tidyverse_1.2.1            usdm_1.1-18                raster_2.8-4               sp_1.3-1                  
## [45] pacman_0.5.0              
## 
## loaded via a namespace (and not attached):
##  [1] readxl_1.2.0       backports_1.1.3    plyr_1.8.4         lazyeval_0.2.1     splines_3.5.1      mycor_0.1.1       
##  [7] crosstalk_1.0.0    leaflet_2.0.2      digest_0.6.18      magrittr_1.5       mosaicCore_0.6.0   openxlsx_4.1.0    
## [13] recipes_0.1.4      modelr_0.1.2       gower_0.1.2        colorspace_1.3-2   rvest_0.3.2        ggrepel_0.8.0     
## [19] haven_2.0.0        crayon_1.3.4       jsonlite_1.5       survival_2.42-3    glue_1.3.0         registry_0.5      
## [25] gtable_0.2.0       ppcor_1.1          ipred_0.9-8        sjmisc_2.7.9       abind_1.4-5        rngtools_1.3.1    
## [31] bibtex_0.4.2       Rcpp_1.0.0         xtable_1.8-3       units_0.6-2        foreign_0.8-70     stats4_3.5.1      
## [37] lava_1.6.4         prodlim_2018.04.18 htmlwidgets_1.3    httr_1.4.0         RColorBrewer_1.1-2 pkgconfig_2.0.2   
## [43] farver_1.1.0       nnet_7.3-12        labeling_0.3       tidyselect_0.2.5   rlang_0.3.1        later_0.7.5       
## [49] munsell_0.5.0      cellranger_1.1.0   tools_3.5.1        cli_1.0.1          generics_0.0.2     moments_0.14      
## [55] sjlabelled_1.0.17  broom_0.5.1        evaluate_0.12      ggdendro_0.1-20    yaml_2.2.0         ModelMetrics_1.2.2
## [61] zip_2.0.1          nlme_3.1-137       doRNG_1.7.1        mime_0.6           xml2_1.2.0         compiler_3.5.1    
## [67] rstudioapi_0.8     curl_3.2           tweenr_1.0.1       stringi_1.2.4      highr_0.7          gdtools_0.1.7     
## [73] pillar_1.3.1       data.table_1.11.8  bitops_1.0-6       insight_0.1.2      httpuv_1.4.5       R6_2.3.0          
## [79] promises_1.0.1     gridExtra_2.3      rio_0.5.16         codetools_0.2-15   assertthat_0.2.0   pkgmaker_0.27     
## [85] withr_2.1.2        nortest_1.0-4      mgcv_1.8-24        hms_0.4.2          quadprog_1.5-5     grid_3.5.1        
## [91] rpart_4.1-13       timeDate_3043.102  class_7.3-14       rmarkdown_1.11     shiny_1.2.0        lubridate_1.7.4